mirror of
https://github.com/open-telemetry/opentelemetry-python-contrib.git
synced 2026-03-13 08:10:39 +08:00
Align gen_ai.tool_definitions with JSON Schema and add it to completion hook (#4181)
* add gen_ai.tool_definitions to completion hook * hash tool defintions * Update CHANGELOG * make tool.defintions optional in on_completion * fix lint errors * add gen_ai.tool_definitions to completion hook * hash tool defintions * fix CompetionHook docstring * fix lint test * add gen_ai.tool_definitions to completion hook * hash tool defintions * Update CHANGELOG * align tool.defintions with JSON schema * fix tests * fix ruff * fix unexpected indent * use local changes from util/opentelemetry-util-genai * fix ruff * add gen_ai.tool_definitions to completion hook * re-add changes lost during merge * put tool.defintions in the telemetry by default and the params behind the flag * fix: Too many local variables * address comments * Add comments to udpate of util-genai version after release --------- Co-authored-by: Aaron Abbott <aaronabbott@google.com>
This commit is contained in:
Wiktoria Walczak
@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
## Unreleased
|
||||
- Fix bug in how tokens are counted when using the streaming `generateContent` method. ([#4152](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4152)).
|
||||
- Add `gen_ai.tool.definitions` attribute to `gen_ai.client.inference.operation.details` log event ([#4142](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4142)).
|
||||
- Add `gen_ai.tool_definitions` to completion hook ([#4181](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4181))
|
||||
|
||||
|
||||
## Version 0.6b0 (2026-01-27)
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ dependencies = [
|
||||
"opentelemetry-api ~=1.37",
|
||||
"opentelemetry-instrumentation >=0.58b0, <2",
|
||||
"opentelemetry-semantic-conventions >=0.58b0, <2",
|
||||
"opentelemetry-util-genai >= 0.2b0, <0.3b0",
|
||||
"opentelemetry-util-genai >= 0.2b0, <0.3b0", # TODO: update version after release (https://github.com/open-telemetry/opentelemetry-python-contrib/issues/4221)
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
||||
@@ -21,14 +21,7 @@ import logging
|
||||
import os
|
||||
import time
|
||||
import typing
|
||||
from typing import (
|
||||
Any,
|
||||
AsyncIterator,
|
||||
Awaitable,
|
||||
Iterator,
|
||||
Optional,
|
||||
Union,
|
||||
)
|
||||
from typing import Any, AsyncIterator, Awaitable, Iterator, Optional, Union
|
||||
|
||||
from google.genai.models import AsyncModels, Models
|
||||
from google.genai.models import t as transformers
|
||||
@@ -65,14 +58,15 @@ from opentelemetry.trace.span import Span
|
||||
from opentelemetry.util.genai.completion_hook import CompletionHook
|
||||
from opentelemetry.util.genai.types import (
|
||||
ContentCapturingMode,
|
||||
FunctionToolDefinition,
|
||||
GenericToolDefinition,
|
||||
InputMessage,
|
||||
MessagePart,
|
||||
OutputMessage,
|
||||
ToolDefinition,
|
||||
)
|
||||
from opentelemetry.util.genai.utils import gen_ai_json_dumps
|
||||
from opentelemetry.util.types import (
|
||||
AttributeValue,
|
||||
)
|
||||
from opentelemetry.util.types import AttributeValue
|
||||
|
||||
from .allowlist_util import AllowList
|
||||
from .custom_semconv import GCP_GENAI_OPERATION_CONFIG
|
||||
@@ -208,68 +202,162 @@ def _to_dict(value: object):
|
||||
return json.loads(json.dumps(value))
|
||||
|
||||
|
||||
def _tool_to_tool_definition(tool: ToolUnionDict) -> MessagePart:
|
||||
def _model_dump_to_tool_definition(tool: Any) -> ToolDefinition:
|
||||
model_dump = tool.model_dump(exclude_none=True)
|
||||
|
||||
name = (
|
||||
model_dump.get("name")
|
||||
or getattr(tool, "name", None)
|
||||
or type(tool).__name__
|
||||
)
|
||||
description = model_dump.get("description") or getattr(
|
||||
tool, "description", None
|
||||
)
|
||||
parameters = model_dump.get("parameters") or model_dump.get("inputSchema")
|
||||
return FunctionToolDefinition(
|
||||
name=name,
|
||||
description=description,
|
||||
parameters=parameters,
|
||||
)
|
||||
|
||||
|
||||
def _clean_parameters(params: Any) -> Any:
|
||||
"""Converts parameter objects into plain dicts."""
|
||||
if params is None:
|
||||
return None
|
||||
if isinstance(params, dict):
|
||||
return params
|
||||
if hasattr(params, "to_dict"):
|
||||
return params.to_dict()
|
||||
if hasattr(params, "model_dump"):
|
||||
return params.model_dump(exclude_none=True)
|
||||
|
||||
try:
|
||||
# Check if it's already a standard JSON type.
|
||||
json.dumps(params)
|
||||
return params
|
||||
|
||||
except (TypeError, ValueError):
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"serialization_error": {
|
||||
"type": "string",
|
||||
"description": f"Failed to serialize parameters: {type(params).__name__}",
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _tool_to_tool_definition(tool: Tool) -> list[ToolDefinition]:
|
||||
definitions = []
|
||||
if tool.function_declarations:
|
||||
for fd in tool.function_declarations:
|
||||
definitions.append(
|
||||
FunctionToolDefinition(
|
||||
name=getattr(fd, "name", type(fd).__name__),
|
||||
description=getattr(fd, "description", None),
|
||||
parameters=_clean_parameters(
|
||||
getattr(fd, "parameters", None)
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
# Generic types
|
||||
if hasattr(tool, "model_dump"):
|
||||
return tool.model_dump(exclude_none=True)
|
||||
exclude_fields = {"function_declarations"}
|
||||
fields = {
|
||||
k: v
|
||||
for k, v in tool.model_dump().items()
|
||||
if v is not None and k not in exclude_fields
|
||||
}
|
||||
|
||||
return str(tool)
|
||||
for tool_type, _ in fields.items():
|
||||
definitions.append(
|
||||
GenericToolDefinition(
|
||||
type=tool_type,
|
||||
name=tool_type,
|
||||
)
|
||||
)
|
||||
|
||||
return definitions
|
||||
|
||||
|
||||
def _callable_tool_to_tool_definition(tool: Any) -> MessagePart:
|
||||
def _callable_tool_to_tool_definition(tool: Any) -> ToolDefinition:
|
||||
doc = getattr(tool, "__doc__", "") or ""
|
||||
return {
|
||||
"name": getattr(tool, "__name__", type(tool).__name__),
|
||||
"description": doc.strip(),
|
||||
}
|
||||
return FunctionToolDefinition(
|
||||
name=getattr(tool, "__name__", type(tool).__name__),
|
||||
description=doc.strip(),
|
||||
parameters=None,
|
||||
)
|
||||
|
||||
|
||||
def _mcp_tool_to_tool_definition(tool: McpTool) -> MessagePart:
|
||||
def _mcp_tool_to_tool_definition(tool: McpTool) -> ToolDefinition:
|
||||
if hasattr(tool, "model_dump"):
|
||||
return tool.model_dump(exclude_none=True)
|
||||
return _model_dump_to_tool_definition(tool)
|
||||
|
||||
return {
|
||||
"name": getattr(tool, "name", type(tool).__name__),
|
||||
"description": getattr(tool, "description", "") or "",
|
||||
"input_schema": getattr(tool, "input_schema", {}),
|
||||
}
|
||||
return FunctionToolDefinition(
|
||||
name=getattr(tool, "name", type(tool).__name__),
|
||||
description=getattr(tool, "description", None),
|
||||
parameters=getattr(tool, "input_schema", None),
|
||||
)
|
||||
|
||||
|
||||
def _to_tool_definition_common(tool: ToolUnionDict) -> MessagePart:
|
||||
if isinstance(tool, dict):
|
||||
return tool
|
||||
|
||||
def _to_tool_definition_common(tool: ToolUnionDict) -> list[ToolDefinition]:
|
||||
if isinstance(tool, Tool):
|
||||
return _tool_to_tool_definition(tool)
|
||||
|
||||
if callable(tool):
|
||||
return _callable_tool_to_tool_definition(tool)
|
||||
return [_callable_tool_to_tool_definition(tool)]
|
||||
|
||||
if _is_mcp_imported and isinstance(tool, McpTool):
|
||||
return _mcp_tool_to_tool_definition(tool)
|
||||
return [_mcp_tool_to_tool_definition(tool)]
|
||||
|
||||
try:
|
||||
return {"raw_definition": json.loads(json.dumps(tool))}
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
return {
|
||||
"error": f"failed to serialize tool definition, tool type={type(tool).__name__}"
|
||||
}
|
||||
return [
|
||||
GenericToolDefinition(
|
||||
name="UnserializableTool",
|
||||
type=type(tool).__name__,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _to_tool_definition(tool: ToolUnionDict) -> MessagePart:
|
||||
def _to_tool_definition(tool: ToolUnionDict) -> list[ToolDefinition]:
|
||||
if _is_mcp_imported and isinstance(tool, McpClientSession):
|
||||
return None
|
||||
return []
|
||||
|
||||
return _to_tool_definition_common(tool)
|
||||
|
||||
|
||||
async def _to_tool_definition_async(tool: ToolUnionDict) -> MessagePart:
|
||||
async def _to_tool_definition_async(
|
||||
tool: ToolUnionDict,
|
||||
) -> list[ToolDefinition]:
|
||||
if _is_mcp_imported and isinstance(tool, McpClientSession):
|
||||
result = await tool.list_tools()
|
||||
return [t.model_dump(exclude_none=True) for t in result.tools]
|
||||
return [_model_dump_to_tool_definition(t) for t in result.tools]
|
||||
|
||||
return _to_tool_definition_common(tool)
|
||||
|
||||
|
||||
def _tool_def_without_parameters_attr(
|
||||
tool_def: list[ToolDefinition],
|
||||
) -> dict[str, AttributeValue]:
|
||||
if tool_def == []:
|
||||
return {}
|
||||
|
||||
return {
|
||||
GEN_AI_TOOL_DEFINITIONS: [
|
||||
dataclasses.asdict(
|
||||
FunctionToolDefinition(
|
||||
name=td.name, description=td.description, parameters=None
|
||||
)
|
||||
if isinstance(td, FunctionToolDefinition)
|
||||
else td
|
||||
)
|
||||
for td in tool_def
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def _create_request_attributes(
|
||||
config: Optional[GenerateContentConfigOrDict],
|
||||
allow_list: AllowList,
|
||||
@@ -385,7 +473,7 @@ def _create_completion_details_attributes(
|
||||
input_messages: list[InputMessage],
|
||||
output_messages: list[OutputMessage],
|
||||
system_instructions: list[MessagePart],
|
||||
tool_definitions: list[MessagePart],
|
||||
tool_definitions: list[ToolDefinition],
|
||||
as_str: bool = False,
|
||||
) -> dict[str, AttributeValue]:
|
||||
attributes: dict[str, AttributeValue] = {
|
||||
@@ -404,7 +492,9 @@ def _create_completion_details_attributes(
|
||||
]
|
||||
|
||||
if tool_definitions:
|
||||
attributes[GEN_AI_TOOL_DEFINITIONS] = tool_definitions
|
||||
attributes[GEN_AI_TOOL_DEFINITIONS] = [
|
||||
dataclasses.asdict(tool_def) for tool_def in tool_definitions
|
||||
]
|
||||
|
||||
return attributes
|
||||
|
||||
@@ -563,44 +653,87 @@ class _GenerateContentInstrumentationHelper:
|
||||
block_reason = response.prompt_feedback.block_reason.name.upper()
|
||||
self._error_type = f"BLOCKED_{block_reason}"
|
||||
|
||||
def _maybe_get_tool_definitions(self, config):
|
||||
def _maybe_get_tool_definitions(self, config) -> list[ToolDefinition]:
|
||||
if (
|
||||
self.sem_conv_opt_in_mode
|
||||
!= _StabilityMode.GEN_AI_LATEST_EXPERIMENTAL
|
||||
):
|
||||
return None
|
||||
return []
|
||||
|
||||
tool_definitions = []
|
||||
if tools := _config_to_tools(config):
|
||||
for tool in tools:
|
||||
definition = _to_tool_definition(tool)
|
||||
if definition is None:
|
||||
continue
|
||||
if isinstance(definition, list):
|
||||
tool_definitions.extend(definition)
|
||||
else:
|
||||
tool_definitions.append(definition)
|
||||
return tool_definitions
|
||||
return [
|
||||
de for tool in tools for de in _to_tool_definition(tool) if de
|
||||
]
|
||||
return []
|
||||
|
||||
async def _maybe_get_tool_definitions_async(self, config):
|
||||
async def _maybe_get_tool_definitions_async(
|
||||
self, config
|
||||
) -> list[ToolDefinition]:
|
||||
if (
|
||||
self.sem_conv_opt_in_mode
|
||||
!= _StabilityMode.GEN_AI_LATEST_EXPERIMENTAL
|
||||
):
|
||||
return None
|
||||
return []
|
||||
|
||||
tool_definitions = []
|
||||
if tools := _config_to_tools(config):
|
||||
for tool in tools:
|
||||
definition = await _to_tool_definition_async(tool)
|
||||
if definition is None:
|
||||
continue
|
||||
if isinstance(definition, list):
|
||||
tool_definitions.extend(definition)
|
||||
else:
|
||||
tool_definitions.append(definition)
|
||||
definitions = await _to_tool_definition_async(tool)
|
||||
for de in definitions:
|
||||
if de:
|
||||
tool_definitions.append(de)
|
||||
|
||||
return tool_definitions
|
||||
|
||||
def _maybe_log_completion_details_in_log(
|
||||
self,
|
||||
event: LogRecord,
|
||||
completion_details_attributes: dict[str, AttributeValue],
|
||||
tool_definitions: Optional[list[ToolDefinition]] = None,
|
||||
):
|
||||
if self._content_recording_enabled in [
|
||||
ContentCapturingMode.EVENT_ONLY,
|
||||
ContentCapturingMode.SPAN_AND_EVENT,
|
||||
]:
|
||||
event.attributes = {
|
||||
**(event.attributes or {}),
|
||||
**completion_details_attributes,
|
||||
}
|
||||
else:
|
||||
event.attributes = {
|
||||
**(event.attributes or {}),
|
||||
**_tool_def_without_parameters_attr(tool_definitions),
|
||||
}
|
||||
|
||||
self._otel_wrapper.log_completion_details(event=event)
|
||||
|
||||
def _maybe_log_completion_details_in_span(
|
||||
self,
|
||||
span: Span,
|
||||
completion_details_attributes: dict[str, AttributeValue],
|
||||
tool_definitions: Optional[list[ToolDefinition]] = None,
|
||||
):
|
||||
if self._content_recording_enabled in [
|
||||
ContentCapturingMode.SPAN_ONLY,
|
||||
ContentCapturingMode.SPAN_AND_EVENT,
|
||||
]:
|
||||
span.set_attributes(
|
||||
{
|
||||
k: gen_ai_json_dumps(v)
|
||||
for k, v in completion_details_attributes.items()
|
||||
}
|
||||
)
|
||||
# request attributes were already set on the span..
|
||||
else:
|
||||
span.set_attributes(
|
||||
{
|
||||
k: gen_ai_json_dumps(v)
|
||||
for k, v in _tool_def_without_parameters_attr(
|
||||
tool_definitions
|
||||
).items()
|
||||
}
|
||||
)
|
||||
|
||||
def _maybe_log_completion_details(
|
||||
self,
|
||||
extra_attributes: dict[str, AttributeValue],
|
||||
@@ -609,7 +742,7 @@ class _GenerateContentInstrumentationHelper:
|
||||
request: Union[ContentListUnion, ContentListUnionDict],
|
||||
candidates: list[Candidate],
|
||||
config: Optional[GenerateContentConfigOrDict] = None,
|
||||
tool_definitions: list[MessagePart] = None,
|
||||
tool_definitions: Optional[list[ToolDefinition]] = None,
|
||||
):
|
||||
if (
|
||||
self.sem_conv_opt_in_mode
|
||||
@@ -633,10 +766,12 @@ class _GenerateContentInstrumentationHelper:
|
||||
| request_attributes
|
||||
| final_attributes,
|
||||
)
|
||||
tool_definitions = tool_definitions or []
|
||||
self.completion_hook.on_completion(
|
||||
inputs=input_messages,
|
||||
outputs=output_messages,
|
||||
system_instruction=system_instructions,
|
||||
tool_definitions=tool_definitions,
|
||||
span=span,
|
||||
log_record=event,
|
||||
)
|
||||
@@ -646,27 +781,16 @@ class _GenerateContentInstrumentationHelper:
|
||||
system_instructions,
|
||||
tool_definitions,
|
||||
)
|
||||
if self._content_recording_enabled in [
|
||||
ContentCapturingMode.EVENT_ONLY,
|
||||
ContentCapturingMode.SPAN_AND_EVENT,
|
||||
]:
|
||||
event.attributes = {
|
||||
**(event.attributes or {}),
|
||||
**completion_details_attributes,
|
||||
}
|
||||
self._otel_wrapper.log_completion_details(event=event)
|
||||
|
||||
if self._content_recording_enabled in [
|
||||
ContentCapturingMode.SPAN_ONLY,
|
||||
ContentCapturingMode.SPAN_AND_EVENT,
|
||||
]:
|
||||
span.set_attributes(
|
||||
{
|
||||
k: gen_ai_json_dumps(v)
|
||||
for k, v in completion_details_attributes.items()
|
||||
}
|
||||
)
|
||||
# request attributes were already set on the span..
|
||||
self._maybe_log_completion_details_in_log(
|
||||
event=event,
|
||||
completion_details_attributes=completion_details_attributes,
|
||||
tool_definitions=tool_definitions,
|
||||
)
|
||||
self._maybe_log_completion_details_in_span(
|
||||
span=span,
|
||||
completion_details_attributes=completion_details_attributes,
|
||||
tool_definitions=tool_definitions,
|
||||
)
|
||||
|
||||
def _maybe_log_system_instruction(
|
||||
self, config: Optional[GenerateContentConfigOrDict] = None
|
||||
|
||||
@@ -21,6 +21,7 @@ import pytest
|
||||
from google.genai.types import (
|
||||
FunctionDeclarationDict,
|
||||
GenerateContentConfig,
|
||||
GoogleMaps,
|
||||
Part,
|
||||
ToolDict,
|
||||
)
|
||||
@@ -35,9 +36,7 @@ from opentelemetry.instrumentation._semconv import (
|
||||
from opentelemetry.instrumentation.google_genai import (
|
||||
GENERATE_CONTENT_EXTRA_ATTRIBUTES_CONTEXT_KEY,
|
||||
)
|
||||
from opentelemetry.semconv._incubating.attributes import (
|
||||
gen_ai_attributes,
|
||||
)
|
||||
from opentelemetry.semconv._incubating.attributes import gen_ai_attributes
|
||||
from opentelemetry.util.genai.types import ContentCapturingMode
|
||||
|
||||
from .base import TestCase
|
||||
@@ -80,7 +79,7 @@ def _mock_mcp_client_session() -> McpClientSession:
|
||||
description="Tool from session",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"properties": {"id": {"type": "integer"}},
|
||||
},
|
||||
)
|
||||
mock_result = create_autospec(McpListToolsResult, instance=True)
|
||||
@@ -106,9 +105,11 @@ def _mock_tool_dict() -> ToolDict:
|
||||
return ToolDict(
|
||||
function_declarations=[
|
||||
FunctionDeclarationDict(
|
||||
name="mock_tool", description="Description of mock tool."
|
||||
name="mock_tool",
|
||||
description="Description of mock tool.",
|
||||
),
|
||||
]
|
||||
],
|
||||
google_maps=GoogleMaps(),
|
||||
)
|
||||
|
||||
|
||||
@@ -470,6 +471,81 @@ class NonStreamingTestCase(TestCase):
|
||||
ContentCapturingMode.NO_CONTENT,
|
||||
ContentCapturingMode.SPAN_ONLY,
|
||||
]:
|
||||
expected_event_attributes = {
|
||||
"TOOL_DEFINITIONS_NO_CONTENT": (
|
||||
{
|
||||
"name": "_mock_callable_tool",
|
||||
"description": "Description of some tool.",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"name": "mock_tool",
|
||||
"description": "Description of mock tool.",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"name": "google_maps",
|
||||
"type": "google_maps",
|
||||
},
|
||||
{
|
||||
"name": "mcp_tool",
|
||||
"description": "A standalone mcp tool",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
),
|
||||
"TOOL_DEFINITIONS_ASYNC_NO_CONTENT": (
|
||||
{
|
||||
"name": "_mock_callable_tool",
|
||||
"description": "Description of some tool.",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"name": "mock_tool",
|
||||
"description": "Description of mock tool.",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"name": "google_maps",
|
||||
"type": "google_maps",
|
||||
},
|
||||
{
|
||||
"name": "mcp_tool",
|
||||
"description": "Tool from session",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"name": "mcp_tool",
|
||||
"description": "A standalone mcp tool",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
),
|
||||
"TOOL_DEFINITIONS_NO_MCP_NO_CONTENT": (
|
||||
{
|
||||
"name": "_mock_callable_tool",
|
||||
"description": "Description of some tool.",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"name": "mock_tool",
|
||||
"description": "Description of mock tool.",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"name": "google_maps",
|
||||
"type": "google_maps",
|
||||
},
|
||||
),
|
||||
}
|
||||
|
||||
self.assertNotIn(
|
||||
gen_ai_attributes.GEN_AI_INPUT_MESSAGES,
|
||||
event.attributes,
|
||||
@@ -482,10 +558,27 @@ class NonStreamingTestCase(TestCase):
|
||||
gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS,
|
||||
event.attributes,
|
||||
)
|
||||
self.assertNotIn(
|
||||
GEN_AI_TOOL_DEFINITIONS,
|
||||
event.attributes,
|
||||
)
|
||||
if _is_mcp_imported:
|
||||
self.assertIn(
|
||||
event.attributes[GEN_AI_TOOL_DEFINITIONS],
|
||||
[
|
||||
expected_event_attributes[
|
||||
"TOOL_DEFINITIONS_NO_CONTENT"
|
||||
],
|
||||
expected_event_attributes[
|
||||
"TOOL_DEFINITIONS_ASYNC_NO_CONTENT"
|
||||
],
|
||||
],
|
||||
)
|
||||
else:
|
||||
self.assertIn(
|
||||
event.attributes[GEN_AI_TOOL_DEFINITIONS],
|
||||
[
|
||||
expected_event_attributes[
|
||||
"TOOL_DEFINITIONS_NO_MCP_NO_CONTENT"
|
||||
],
|
||||
],
|
||||
)
|
||||
|
||||
else:
|
||||
expected_event_attributes = {
|
||||
@@ -513,72 +606,87 @@ class NonStreamingTestCase(TestCase):
|
||||
{
|
||||
"name": "_mock_callable_tool",
|
||||
"description": "Description of some tool.",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"function_declarations": (
|
||||
{
|
||||
"name": "mock_tool",
|
||||
"description": "Description of mock tool.",
|
||||
},
|
||||
)
|
||||
"name": "mock_tool",
|
||||
"description": "Description of mock tool.",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"name": "google_maps",
|
||||
"type": "google_maps",
|
||||
},
|
||||
{
|
||||
"name": "mcp_tool",
|
||||
"description": "A standalone mcp tool",
|
||||
"inputSchema": {
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "integer"}
|
||||
},
|
||||
},
|
||||
"type": "function",
|
||||
},
|
||||
),
|
||||
"TOOL_DEFINITIONS_ASYNC": (
|
||||
{
|
||||
"name": "_mock_callable_tool",
|
||||
"description": "Description of some tool.",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"function_declarations": (
|
||||
{
|
||||
"name": "mock_tool",
|
||||
"description": "Description of mock tool.",
|
||||
},
|
||||
)
|
||||
"name": "mock_tool",
|
||||
"description": "Description of mock tool.",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"name": "google_maps",
|
||||
"type": "google_maps",
|
||||
},
|
||||
{
|
||||
"name": "mcp_tool",
|
||||
"description": "Tool from session",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"}
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "mcp_tool",
|
||||
"description": "A standalone mcp tool",
|
||||
"inputSchema": {
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "integer"}
|
||||
},
|
||||
},
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"name": "mcp_tool",
|
||||
"description": "A standalone mcp tool",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "integer"}
|
||||
},
|
||||
},
|
||||
"type": "function",
|
||||
},
|
||||
),
|
||||
"TOOL_DEFINITIONS_NO_MCP": (
|
||||
{
|
||||
"name": "_mock_callable_tool",
|
||||
"description": "Description of some tool.",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"function_declarations": (
|
||||
{
|
||||
"name": "mock_tool",
|
||||
"description": "Description of mock tool.",
|
||||
},
|
||||
)
|
||||
"name": "mock_tool",
|
||||
"description": "Description of mock tool.",
|
||||
"parameters": None,
|
||||
"type": "function",
|
||||
},
|
||||
{
|
||||
"name": "google_maps",
|
||||
"type": "google_maps",
|
||||
},
|
||||
),
|
||||
}
|
||||
@@ -696,14 +804,14 @@ class NonStreamingTestCase(TestCase):
|
||||
self.assertIn(
|
||||
span.attributes[GEN_AI_TOOL_DEFINITIONS],
|
||||
[
|
||||
'[{"name":"_mock_callable_tool","description":"Description of some tool."},{"function_declarations":[{"description":"Description of mock tool.","name":"mock_tool"}]},{"name":"mcp_tool","description":"Tool from session","inputSchema":{"type":"object","properties":{"query":{"type":"string"}}}},{"name":"mcp_tool","description":"A standalone mcp tool","inputSchema":{"type":"object","properties":{"id":{"type":"integer"}}}}]',
|
||||
'[{"name":"_mock_callable_tool","description":"Description of some tool."},{"function_declarations":[{"description":"Description of mock tool.","name":"mock_tool"}]},{"name":"mcp_tool","description":"A standalone mcp tool","inputSchema":{"type":"object","properties":{"id":{"type":"integer"}}}}]',
|
||||
'[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"},{"name":"mcp_tool","description":"Tool from session","parameters":{"type":"object","properties":{"id":{"type":"integer"}}},"type":"function"},{"name":"mcp_tool","description":"A standalone mcp tool","parameters":{"type":"object","properties":{"id":{"type":"integer"}}},"type":"function"}]',
|
||||
'[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"},{"name":"mcp_tool","description":"A standalone mcp tool","parameters":{"type":"object","properties":{"id":{"type":"integer"}}},"type":"function"}]',
|
||||
],
|
||||
)
|
||||
else:
|
||||
self.assertEqual(
|
||||
span.attributes[GEN_AI_TOOL_DEFINITIONS],
|
||||
'[{"name":"_mock_callable_tool","description":"Description of some tool."},{"function_declarations":[{"description":"Description of mock tool.","name":"mock_tool"}]}]',
|
||||
'[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"}]',
|
||||
)
|
||||
else:
|
||||
self.assertNotIn(
|
||||
@@ -718,10 +826,19 @@ class NonStreamingTestCase(TestCase):
|
||||
gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS,
|
||||
span.attributes,
|
||||
)
|
||||
self.assertNotIn(
|
||||
GEN_AI_TOOL_DEFINITIONS,
|
||||
span.attributes,
|
||||
)
|
||||
if _is_mcp_imported:
|
||||
self.assertIn(
|
||||
span.attributes[GEN_AI_TOOL_DEFINITIONS],
|
||||
[
|
||||
'[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"},{"name":"mcp_tool","description":"Tool from session","parameters":null,"type":"function"},{"name":"mcp_tool","description":"A standalone mcp tool","parameters":null,"type":"function"}]',
|
||||
'[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"},{"name":"mcp_tool","description":"A standalone mcp tool","parameters":null,"type":"function"}]',
|
||||
],
|
||||
)
|
||||
else:
|
||||
self.assertEqual(
|
||||
span.attributes[GEN_AI_TOOL_DEFINITIONS],
|
||||
'[{"name":"_mock_callable_tool","description":"Description of some tool.","parameters":null,"type":"function"},{"name":"mock_tool","description":"Description of mock tool.","parameters":null,"type":"function"},{"name":"google_maps","type":"google_maps"}]',
|
||||
)
|
||||
|
||||
self.tearDown()
|
||||
|
||||
|
||||
@@ -25,10 +25,13 @@ opentelemetry-api==1.37.0
|
||||
opentelemetry-sdk==1.37.0
|
||||
opentelemetry-semantic-conventions==0.58b0
|
||||
opentelemetry-instrumentation==0.58b0
|
||||
opentelemetry-util-genai[upload]==0.2b0
|
||||
# TODO: uncomment and update version after release (https://github.com/open-telemetry/opentelemetry-python-contrib/issues/4221)
|
||||
# opentelemetry-util-genai[upload]==0.2b0
|
||||
|
||||
fsspec==2025.9.0
|
||||
|
||||
# Install locally from the folder. This path is relative to the
|
||||
# root directory, given invocation from "tox" at root level.
|
||||
-e instrumentation-genai/opentelemetry-instrumentation-google-genai
|
||||
# TODO: remove local install after release (https://github.com/open-telemetry/opentelemetry-python-contrib/issues/4221)
|
||||
-e util/opentelemetry-util-genai[upload]
|
||||
|
||||
@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## Unreleased
|
||||
|
||||
- Add `gen_ai.tool_definitions` to completion hook ([#4181](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4181))
|
||||
- Add support for emitting inference events and enrich message types. ([#3994](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3994))
|
||||
- Add support for `server.address`, `server.port` on all signals and additional metric-only attributes
|
||||
([#4069](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/4069))
|
||||
|
||||
@@ -15,7 +15,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import posixpath
|
||||
import threading
|
||||
@@ -50,6 +52,11 @@ GEN_AI_SYSTEM_INSTRUCTIONS_REF: Final = (
|
||||
gen_ai_attributes.GEN_AI_SYSTEM_INSTRUCTIONS + "_ref"
|
||||
)
|
||||
|
||||
GEN_AI_TOOL_DEFINITIONS = getattr(
|
||||
gen_ai_attributes, "GEN_AI_TOOL_DEFINITIONS", "gen_ai.tool.definitions"
|
||||
)
|
||||
GEN_AI_TOOL_DEFINITIONS_REF: Final = GEN_AI_TOOL_DEFINITIONS + "_ref"
|
||||
|
||||
_MESSAGE_INDEX_KEY = "index"
|
||||
_DEFAULT_MAX_QUEUE_SIZE = 20
|
||||
_DEFAULT_FORMAT = "json"
|
||||
@@ -66,6 +73,7 @@ class Completion:
|
||||
inputs: list[types.InputMessage] | None
|
||||
outputs: list[types.OutputMessage] | None
|
||||
system_instruction: list[types.MessagePart] | None
|
||||
tool_definitions: list[types.ToolDefinition] | None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -73,6 +81,7 @@ class CompletionRefs:
|
||||
inputs_ref: str
|
||||
outputs_ref: str
|
||||
system_instruction_ref: str
|
||||
tool_definitions_ref: str
|
||||
|
||||
|
||||
JsonEncodeable = list[dict[str, Any]]
|
||||
@@ -81,14 +90,39 @@ JsonEncodeable = list[dict[str, Any]]
|
||||
UploadData = dict[tuple[str, bool], Callable[[], JsonEncodeable]]
|
||||
|
||||
|
||||
def is_system_instructions_hashable(
|
||||
system_instruction: list[types.MessagePart] | None,
|
||||
def is_message_part_list_hashable(
|
||||
message_parts: list[types.MessagePart] | None,
|
||||
) -> bool:
|
||||
return bool(system_instruction) and all(
|
||||
isinstance(x, types.Text) for x in system_instruction
|
||||
return bool(message_parts) and all(
|
||||
isinstance(x, types.Text) for x in message_parts
|
||||
)
|
||||
|
||||
|
||||
def hash_tool_definitions(
|
||||
tool_definitions: list[types.ToolDefinition] | None,
|
||||
) -> str | None:
|
||||
if not tool_definitions:
|
||||
return None
|
||||
try:
|
||||
tool_dicts = [
|
||||
{k: v for k, v in dataclasses.asdict(t).items() if v is not None}
|
||||
for t in tool_definitions
|
||||
]
|
||||
|
||||
encoded_tools = json.dumps(
|
||||
tool_dicts,
|
||||
sort_keys=True,
|
||||
).encode("utf-8")
|
||||
|
||||
return hashlib.sha256(
|
||||
encoded_tools,
|
||||
usedforsecurity=False,
|
||||
).hexdigest()
|
||||
|
||||
except (TypeError, AttributeError):
|
||||
return None
|
||||
|
||||
|
||||
class UploadCompletionHook(CompletionHook):
|
||||
"""An completion hook using ``fsspec`` to upload to external storage
|
||||
|
||||
@@ -169,12 +203,14 @@ class UploadCompletionHook(CompletionHook):
|
||||
self._semaphore.release()
|
||||
|
||||
def _calculate_ref_path(
|
||||
self, system_instruction: list[types.MessagePart]
|
||||
self,
|
||||
system_instruction: list[types.MessagePart],
|
||||
tool_definitions: list[types.ToolDefinition] | None = None,
|
||||
) -> CompletionRefs:
|
||||
# TODO: experimental with using the trace_id and span_id, or fetching
|
||||
# gen_ai.response.id from the active span.
|
||||
system_instruction_hash = None
|
||||
if is_system_instructions_hashable(system_instruction):
|
||||
if is_message_part_list_hashable(system_instruction):
|
||||
# Get a hash of the text.
|
||||
system_instruction_hash = hashlib.sha256(
|
||||
"\n".join(x.content for x in system_instruction).encode( # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue, reportUnknownArgumentType, reportCallIssue, reportArgumentType]
|
||||
@@ -182,6 +218,9 @@ class UploadCompletionHook(CompletionHook):
|
||||
),
|
||||
usedforsecurity=False,
|
||||
).hexdigest()
|
||||
|
||||
tool_definitions_hash = hash_tool_definitions(tool_definitions)
|
||||
|
||||
uuid_str = str(uuid4())
|
||||
return CompletionRefs(
|
||||
inputs_ref=posixpath.join(
|
||||
@@ -194,6 +233,10 @@ class UploadCompletionHook(CompletionHook):
|
||||
self._base_path,
|
||||
f"{system_instruction_hash or uuid_str}_system_instruction.{self._format}",
|
||||
),
|
||||
tool_definitions_ref=posixpath.join(
|
||||
self._base_path,
|
||||
f"{tool_definitions_hash or uuid_str}_tool.definitions.{self._format}",
|
||||
),
|
||||
)
|
||||
|
||||
def _file_exists(self, path: str) -> bool:
|
||||
@@ -250,25 +293,30 @@ class UploadCompletionHook(CompletionHook):
|
||||
inputs: list[types.InputMessage],
|
||||
outputs: list[types.OutputMessage],
|
||||
system_instruction: list[types.MessagePart],
|
||||
tool_definitions: list[types.ToolDefinition] | None = None,
|
||||
span: Span | None = None,
|
||||
log_record: LogRecord | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if not any([inputs, outputs, system_instruction]):
|
||||
if not any([inputs, outputs, system_instruction, tool_definitions]):
|
||||
return
|
||||
# An empty list will not be uploaded.
|
||||
completion = Completion(
|
||||
inputs=inputs or None,
|
||||
outputs=outputs or None,
|
||||
system_instruction=system_instruction or None,
|
||||
tool_definitions=tool_definitions or None,
|
||||
)
|
||||
# generate the paths to upload to
|
||||
ref_names = self._calculate_ref_path(system_instruction)
|
||||
ref_names = self._calculate_ref_path(
|
||||
system_instruction, tool_definitions
|
||||
)
|
||||
|
||||
def to_dict(
|
||||
dataclass_list: list[types.InputMessage]
|
||||
| list[types.OutputMessage]
|
||||
| list[types.MessagePart],
|
||||
| list[types.MessagePart]
|
||||
| list[types.ToolDefinition],
|
||||
) -> JsonEncodeable:
|
||||
return [asdict(dc) for dc in dataclass_list]
|
||||
|
||||
@@ -291,12 +339,18 @@ class UploadCompletionHook(CompletionHook):
|
||||
ref_names.system_instruction_ref,
|
||||
completion.system_instruction,
|
||||
GEN_AI_SYSTEM_INSTRUCTIONS_REF,
|
||||
is_system_instructions_hashable(
|
||||
is_message_part_list_hashable(
|
||||
completion.system_instruction
|
||||
),
|
||||
),
|
||||
(
|
||||
ref_names.tool_definitions_ref,
|
||||
completion.tool_definitions,
|
||||
GEN_AI_TOOL_DEFINITIONS_REF,
|
||||
bool(completion.tool_definitions),
|
||||
),
|
||||
]
|
||||
if ref # Filter out empty input/output/sys instruction
|
||||
if ref # Filter out empty input/output/sys instruction/tool defs
|
||||
]
|
||||
self._submit_all(
|
||||
{
|
||||
|
||||
@@ -61,6 +61,8 @@ class CompletionHook(Protocol):
|
||||
outputs: The outputs of the GenAI interaction.
|
||||
system_instruction: The system instruction of the GenAI
|
||||
interaction.
|
||||
tool_definitions: The list of source system tool definitions
|
||||
available to the GenAI agent or model.
|
||||
span: The span associated with the GenAI interaction.
|
||||
log_record: The event log associated with the GenAI
|
||||
interaction.
|
||||
@@ -72,6 +74,7 @@ class CompletionHook(Protocol):
|
||||
inputs: list[types.InputMessage],
|
||||
outputs: list[types.OutputMessage],
|
||||
system_instruction: list[types.MessagePart],
|
||||
tool_definitions: list[types.ToolDefinition] | None = None,
|
||||
span: Span | None = None,
|
||||
log_record: LogRecord | None = None,
|
||||
) -> None: ...
|
||||
|
||||
@@ -137,6 +137,26 @@ class Uri:
|
||||
type: Literal["uri"] = "uri"
|
||||
|
||||
|
||||
@dataclass()
|
||||
class FunctionToolDefinition:
|
||||
"""Represents a function tool definition sent to the model"""
|
||||
|
||||
name: str
|
||||
description: str | None
|
||||
parameters: Any
|
||||
type: Literal["function"] = "function"
|
||||
|
||||
|
||||
@dataclass()
|
||||
class GenericToolDefinition:
|
||||
"""Represents a generic tool definition sent to the model"""
|
||||
|
||||
name: str
|
||||
type: str
|
||||
|
||||
|
||||
ToolDefinition = Union[FunctionToolDefinition, GenericToolDefinition]
|
||||
|
||||
MessagePart = Union[
|
||||
Text, ToolCall, ToolCallResponse, Blob, File, Uri, Reasoning, Any
|
||||
]
|
||||
|
||||
@@ -69,6 +69,15 @@ FAKE_OUTPUTS = [
|
||||
]
|
||||
FAKE_SYSTEM_INSTRUCTION = [types.Text(content="You are a helpful assistant.")]
|
||||
|
||||
FAKE_TOOL_DEFINITIONS: list[types.ToolDefinition] = [
|
||||
types.FunctionToolDefinition(
|
||||
name="test_tool",
|
||||
description="does something",
|
||||
parameters=None,
|
||||
type="function",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class ThreadSafeMagicMock(MagicMock):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
@@ -124,6 +133,7 @@ class TestUploadCompletionHook(TestCase):
|
||||
inputs=FAKE_INPUTS,
|
||||
outputs=FAKE_OUTPUTS,
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=FAKE_TOOL_DEFINITIONS,
|
||||
)
|
||||
# all items should be consumed
|
||||
self.hook.shutdown()
|
||||
@@ -131,8 +141,8 @@ class TestUploadCompletionHook(TestCase):
|
||||
time.sleep(0.5)
|
||||
self.assertEqual(
|
||||
self.mock_fs.open.call_count,
|
||||
3,
|
||||
"should have uploaded 3 files",
|
||||
4,
|
||||
"should have uploaded 4 files",
|
||||
)
|
||||
|
||||
def test_lru_cache_works(self):
|
||||
@@ -141,6 +151,7 @@ class TestUploadCompletionHook(TestCase):
|
||||
inputs=[],
|
||||
outputs=[],
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=[],
|
||||
log_record=record,
|
||||
)
|
||||
# Wait a bit for file upload to finish..
|
||||
@@ -161,6 +172,7 @@ class TestUploadCompletionHook(TestCase):
|
||||
inputs=[],
|
||||
outputs=[],
|
||||
system_instruction=[types.Text(content=str(iteration))],
|
||||
tool_definitions=[],
|
||||
)
|
||||
self.hook.shutdown()
|
||||
self.assertFalse(
|
||||
@@ -174,6 +186,7 @@ class TestUploadCompletionHook(TestCase):
|
||||
inputs=[],
|
||||
outputs=[],
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=[],
|
||||
log_record=record,
|
||||
)
|
||||
# all items should be consumed
|
||||
@@ -203,6 +216,7 @@ class TestUploadCompletionHook(TestCase):
|
||||
inputs=FAKE_INPUTS,
|
||||
outputs=FAKE_OUTPUTS,
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=FAKE_TOOL_DEFINITIONS,
|
||||
)
|
||||
|
||||
self.assertLessEqual(
|
||||
@@ -216,6 +230,7 @@ class TestUploadCompletionHook(TestCase):
|
||||
inputs=FAKE_INPUTS,
|
||||
outputs=FAKE_OUTPUTS,
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=FAKE_TOOL_DEFINITIONS,
|
||||
)
|
||||
|
||||
self.assertIn(
|
||||
@@ -228,6 +243,7 @@ class TestUploadCompletionHook(TestCase):
|
||||
inputs=FAKE_INPUTS,
|
||||
outputs=FAKE_OUTPUTS,
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=FAKE_TOOL_DEFINITIONS,
|
||||
)
|
||||
|
||||
# shutdown should timeout and return even though there are still items in the queue
|
||||
@@ -241,6 +257,7 @@ class TestUploadCompletionHook(TestCase):
|
||||
inputs=FAKE_INPUTS,
|
||||
outputs=FAKE_OUTPUTS,
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=FAKE_TOOL_DEFINITIONS,
|
||||
)
|
||||
self.hook.shutdown()
|
||||
|
||||
@@ -264,6 +281,7 @@ class TestUploadCompletionHook(TestCase):
|
||||
inputs=FAKE_INPUTS,
|
||||
outputs=FAKE_OUTPUTS,
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=FAKE_TOOL_DEFINITIONS,
|
||||
)
|
||||
hook.shutdown()
|
||||
|
||||
@@ -278,8 +296,9 @@ class TestUploadCompletionHook(TestCase):
|
||||
inputs=FAKE_INPUTS,
|
||||
outputs=FAKE_OUTPUTS,
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=FAKE_TOOL_DEFINITIONS,
|
||||
)
|
||||
self.assertEqual(len(logs.output), 3)
|
||||
self.assertEqual(len(logs.output), 4)
|
||||
self.assertIn(
|
||||
"attempting to upload file after UploadCompletionHook.shutdown() was already called",
|
||||
logs.output[0],
|
||||
@@ -337,6 +356,7 @@ class TestUploadCompletionHookIntegration(TestBase):
|
||||
inputs=[],
|
||||
outputs=[],
|
||||
system_instruction=system_instructions,
|
||||
tool_definitions=[],
|
||||
log_record=record,
|
||||
)
|
||||
self.hook.shutdown()
|
||||
@@ -349,6 +369,42 @@ class TestUploadCompletionHookIntegration(TestBase):
|
||||
# Content should not have been overwritten.
|
||||
self.assert_fsspec_equal(expected_file_name, "asg")
|
||||
|
||||
def test_tool_definitions_is_hashed_to_avoid_reupload(self):
|
||||
expected_hash = (
|
||||
"1f559d0102f8c440a667fd5ed587beeed488ec9f3ce0828d39c424bed6546cf5"
|
||||
)
|
||||
# Create the file before upload..
|
||||
expected_file_name = f"memory://{expected_hash}_tool.definitions.json"
|
||||
with fsspec.open(expected_file_name, "wb") as file:
|
||||
file.write(b"asg")
|
||||
# FIle should exist.
|
||||
self.assertTrue(self.hook._file_exists(expected_file_name))
|
||||
tool_definitions = [
|
||||
types.FunctionToolDefinition(
|
||||
name="some_tool",
|
||||
description="does something",
|
||||
parameters=None,
|
||||
type="function",
|
||||
),
|
||||
]
|
||||
record = LogRecord()
|
||||
self.hook.on_completion(
|
||||
inputs=[],
|
||||
outputs=[],
|
||||
system_instruction=[],
|
||||
tool_definitions=tool_definitions,
|
||||
log_record=record,
|
||||
)
|
||||
self.hook.shutdown()
|
||||
self.assertIsNotNone(record.attributes)
|
||||
|
||||
self.assertEqual(
|
||||
record.attributes["gen_ai.tool.definitions_ref"],
|
||||
expected_file_name,
|
||||
)
|
||||
# Content should not have been overwritten.
|
||||
self.assert_fsspec_equal(expected_file_name, "asg")
|
||||
|
||||
def test_upload_completions(self):
|
||||
tracer = self.tracer_provider.get_tracer(__name__)
|
||||
log_record = LogRecord()
|
||||
@@ -358,6 +414,7 @@ class TestUploadCompletionHookIntegration(TestBase):
|
||||
inputs=FAKE_INPUTS,
|
||||
outputs=FAKE_OUTPUTS,
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=FAKE_TOOL_DEFINITIONS,
|
||||
span=span,
|
||||
log_record=log_record,
|
||||
)
|
||||
@@ -376,6 +433,7 @@ class TestUploadCompletionHookIntegration(TestBase):
|
||||
"gen_ai.input.messages_ref",
|
||||
"gen_ai.output.messages_ref",
|
||||
"gen_ai.system_instructions_ref",
|
||||
"gen_ai.tool.definitions_ref",
|
||||
]:
|
||||
self.assertIn(ref_key, attributes)
|
||||
|
||||
@@ -391,6 +449,10 @@ class TestUploadCompletionHookIntegration(TestBase):
|
||||
span.attributes["gen_ai.system_instructions_ref"],
|
||||
'[{"content":"You are a helpful assistant.","type":"text"}]\n',
|
||||
)
|
||||
self.assert_fsspec_equal(
|
||||
span.attributes["gen_ai.tool.definitions_ref"],
|
||||
'[{"name":"test_tool","description":"does something","parameters":null,"type":"function"}]\n',
|
||||
)
|
||||
|
||||
def test_stamps_empty_log(self):
|
||||
log_record = LogRecord()
|
||||
@@ -398,6 +460,7 @@ class TestUploadCompletionHookIntegration(TestBase):
|
||||
inputs=FAKE_INPUTS,
|
||||
outputs=FAKE_OUTPUTS,
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=FAKE_TOOL_DEFINITIONS,
|
||||
log_record=log_record,
|
||||
)
|
||||
|
||||
@@ -405,6 +468,7 @@ class TestUploadCompletionHookIntegration(TestBase):
|
||||
self.assertIn("gen_ai.input.messages_ref", log_record.attributes)
|
||||
self.assertIn("gen_ai.output.messages_ref", log_record.attributes)
|
||||
self.assertIn("gen_ai.system_instructions_ref", log_record.attributes)
|
||||
self.assertIn("gen_ai.tool.definitions_ref", log_record.attributes)
|
||||
|
||||
def test_upload_bytes(self) -> None:
|
||||
log_record = LogRecord()
|
||||
@@ -420,6 +484,7 @@ class TestUploadCompletionHookIntegration(TestBase):
|
||||
],
|
||||
outputs=FAKE_OUTPUTS,
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=FAKE_TOOL_DEFINITIONS,
|
||||
log_record=log_record,
|
||||
)
|
||||
self.hook.shutdown()
|
||||
@@ -438,6 +503,7 @@ class TestUploadCompletionHookIntegration(TestBase):
|
||||
inputs=FAKE_INPUTS,
|
||||
outputs=FAKE_OUTPUTS,
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=FAKE_TOOL_DEFINITIONS,
|
||||
log_record=log_record,
|
||||
)
|
||||
hook.shutdown()
|
||||
@@ -461,6 +527,7 @@ class TestUploadCompletionHookIntegration(TestBase):
|
||||
inputs=FAKE_INPUTS,
|
||||
outputs=FAKE_OUTPUTS,
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=FAKE_TOOL_DEFINITIONS,
|
||||
log_record=log_record,
|
||||
)
|
||||
hook.shutdown()
|
||||
@@ -492,6 +559,7 @@ class TestUploadCompletionHookIntegration(TestBase):
|
||||
inputs=FAKE_INPUTS,
|
||||
outputs=FAKE_OUTPUTS,
|
||||
system_instruction=FAKE_SYSTEM_INSTRUCTION,
|
||||
tool_definitions=FAKE_TOOL_DEFINITIONS,
|
||||
log_record=log_record,
|
||||
)
|
||||
hook.shutdown()
|
||||
|
||||
Reference in New Issue
Block a user