mirror of
https://github.com/open-telemetry/opentelemetry-python-contrib.git
synced 2026-03-13 08:10:39 +08:00
Update vertex AI instrumentation to handle InlineData and FileData parts (#3840)
* Initial commit * Update changelog
This commit is contained in:
DylanRussell
@@ -14,8 +14,10 @@ users will need to set the environment variable OTEL_SEMCONV_STABILITY_OPT_IN to
|
||||
([#3328](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3328))
|
||||
- VertexAI support for async calling
|
||||
([#3386](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3386))
|
||||
- `opentelemetry-instrumentation-vertexai`: migrate off the deprecated events API to use the logs API
|
||||
- Migrate off the deprecated events API to use the logs API
|
||||
([#3625](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3626))
|
||||
- Update `gen_ai_latest_experimental` instrumentation to record files being passed to the model
|
||||
([#3840](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3840)).
|
||||
|
||||
## Version 2.0b0 (2025-02-24)
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from os import environ
|
||||
@@ -308,6 +309,23 @@ def request_to_events(
|
||||
yield user_event(role=content.role, content=request_content)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BlobPart:
|
||||
data: bytes
|
||||
mime_type: str
|
||||
type: Literal["blob"] = "blob"
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileDataPart:
|
||||
mime_type: str
|
||||
uri: str
|
||||
type: Literal["file_data"] = "file_data"
|
||||
|
||||
class Config:
|
||||
extra = "allow"
|
||||
|
||||
|
||||
def convert_content_to_message_parts(
|
||||
content: content.Content | content_v1beta1.Content,
|
||||
) -> list[MessagePart]:
|
||||
@@ -334,12 +352,20 @@ def convert_content_to_message_parts(
|
||||
)
|
||||
elif "text" in part:
|
||||
parts.append(Text(content=part.text))
|
||||
else:
|
||||
dict_part = type(part).to_dict( # type: ignore[reportUnknownMemberType]
|
||||
part, always_print_fields_with_no_presence=False
|
||||
elif "inline_data" in part:
|
||||
part = part.inline_data
|
||||
parts.append(
|
||||
BlobPart(mime_type=part.mime_type or "", data=part.data or b"")
|
||||
)
|
||||
dict_part["type"] = type(part)
|
||||
parts.append(dict_part)
|
||||
elif "file_data" in part:
|
||||
part = part.file_data
|
||||
parts.append(
|
||||
FileDataPart(
|
||||
mime_type=part.mime_type or "", uri=part.file_uri or ""
|
||||
)
|
||||
)
|
||||
else:
|
||||
logging.warning("Unknown part dropped from telemetry %s", part)
|
||||
return parts
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: |-
|
||||
{
|
||||
"contents": [
|
||||
{
|
||||
"role": "user",
|
||||
"parts": [
|
||||
{
|
||||
"text": "Say this is a test"
|
||||
},
|
||||
{
|
||||
"fileData": {
|
||||
"mimeType": "image/jpeg",
|
||||
"fileUri": "https://images.pdimagearchive.org/collections/microscopic-delights/1lede-0021.jpg"
|
||||
}
|
||||
},
|
||||
{
|
||||
"inlineData": {
|
||||
"mimeType": "image/jpeg",
|
||||
"data": "iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg=="
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
headers:
|
||||
Accept:
|
||||
- '*/*'
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Length:
|
||||
- '554'
|
||||
Content-Type:
|
||||
- application/json
|
||||
User-Agent:
|
||||
- python-requests/2.32.3
|
||||
method: POST
|
||||
uri: https://us-central1-aiplatform.googleapis.com/v1/projects/fake-project/locations/us-central1/publishers/google/models/gemini-2.5-pro:generateContent?%24alt=json%3Benum-encoding%3Dint
|
||||
response:
|
||||
body:
|
||||
string: |-
|
||||
{
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"role": "model",
|
||||
"parts": [
|
||||
{
|
||||
"text": "This is a test."
|
||||
}
|
||||
]
|
||||
},
|
||||
"finishReason": 1,
|
||||
"avgLogprobs": -24.462081909179688
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 521,
|
||||
"candidatesTokenCount": 5,
|
||||
"totalTokenCount": 950,
|
||||
"trafficType": 1,
|
||||
"promptTokensDetails": [
|
||||
{
|
||||
"modality": 2,
|
||||
"tokenCount": 516
|
||||
},
|
||||
{
|
||||
"modality": 1,
|
||||
"tokenCount": 5
|
||||
}
|
||||
],
|
||||
"candidatesTokensDetails": [
|
||||
{
|
||||
"modality": 1,
|
||||
"tokenCount": 5
|
||||
}
|
||||
],
|
||||
"thoughtsTokenCount": 424
|
||||
},
|
||||
"modelVersion": "gemini-2.5-pro",
|
||||
"createTime": "2025-10-13T16:29:47.639271Z",
|
||||
"responseId": "-yjtaKeCJ5KYmecP76S4-AI"
|
||||
}
|
||||
headers:
|
||||
Content-Type:
|
||||
- application/json; charset=UTF-8
|
||||
Transfer-Encoding:
|
||||
- chunked
|
||||
Vary:
|
||||
- Origin
|
||||
- X-Origin
|
||||
- Referer
|
||||
content-length:
|
||||
- '808'
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -6,6 +6,7 @@ from vertexai.generative_models import (
|
||||
Content,
|
||||
GenerationConfig,
|
||||
GenerativeModel,
|
||||
Image,
|
||||
Part,
|
||||
)
|
||||
from vertexai.preview.generative_models import (
|
||||
@@ -24,7 +25,7 @@ from opentelemetry.trace import StatusCode
|
||||
|
||||
|
||||
@pytest.mark.vcr()
|
||||
def test_generate_content(
|
||||
def test_generate_content_with_files(
|
||||
span_exporter: InMemorySpanExporter,
|
||||
log_exporter: InMemoryLogExporter,
|
||||
generate_content: callable,
|
||||
@@ -38,6 +39,15 @@ def test_generate_content(
|
||||
role="user",
|
||||
parts=[
|
||||
Part.from_text("Say this is a test"),
|
||||
Part.from_uri(
|
||||
mime_type="image/jpeg",
|
||||
uri="https://images.pdimagearchive.org/collections/microscopic-delights/1lede-0021.jpg",
|
||||
),
|
||||
Part.from_image(
|
||||
Image.from_bytes(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg=="
|
||||
)
|
||||
),
|
||||
],
|
||||
),
|
||||
],
|
||||
@@ -52,11 +62,11 @@ def test_generate_content(
|
||||
"gen_ai.request.model": "gemini-2.5-pro",
|
||||
"gen_ai.response.finish_reasons": ("stop",),
|
||||
"gen_ai.response.model": "gemini-2.5-pro",
|
||||
"gen_ai.usage.input_tokens": 5,
|
||||
"gen_ai.usage.input_tokens": 521,
|
||||
"gen_ai.usage.output_tokens": 5,
|
||||
"server.address": "us-central1-aiplatform.googleapis.com",
|
||||
"server.port": 443,
|
||||
"gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"}]}]',
|
||||
"gen_ai.input.messages": '[{"role":"user","parts":[{"content":"Say this is a test","type":"text"},{"mime_type":"image/jpeg","uri":"https://images.pdimagearchive.org/collections/microscopic-delights/1lede-0021.jpg","type":"file_data"},{"data":"iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==","mime_type":"image/jpeg","type":"blob"}]}]',
|
||||
"gen_ai.output.messages": '[{"role":"model","parts":[{"content":"This is a test.","type":"text"}],"finish_reason":"stop"}]',
|
||||
}
|
||||
|
||||
@@ -64,24 +74,36 @@ def test_generate_content(
|
||||
assert len(logs) == 1
|
||||
log = logs[0].log_record
|
||||
assert log.attributes == {
|
||||
"gen_ai.operation.name": "chat",
|
||||
"gen_ai.request.model": "gemini-2.5-pro",
|
||||
"server.address": "us-central1-aiplatform.googleapis.com",
|
||||
"server.port": 443,
|
||||
"gen_ai.operation.name": "chat",
|
||||
"gen_ai.request.model": "gemini-2.5-pro",
|
||||
"gen_ai.response.model": "gemini-2.5-pro",
|
||||
"gen_ai.response.finish_reasons": ("stop",),
|
||||
"gen_ai.usage.input_tokens": 5,
|
||||
"gen_ai.usage.input_tokens": 521,
|
||||
"gen_ai.usage.output_tokens": 5,
|
||||
"gen_ai.input.messages": (
|
||||
{
|
||||
"role": "user",
|
||||
"parts": ({"type": "text", "content": "Say this is a test"},),
|
||||
"parts": (
|
||||
{"content": "Say this is a test", "type": "text"},
|
||||
{
|
||||
"mime_type": "image/jpeg",
|
||||
"uri": "https://images.pdimagearchive.org/collections/microscopic-delights/1lede-0021.jpg",
|
||||
"type": "file_data",
|
||||
},
|
||||
{
|
||||
"data": b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x05\x00\x00\x00\x05\x08\x06\x00\x00\x00\x8do&\xe5\x00\x00\x00\x1cIDAT\x08\xd7c\xf8\xff\xff?\xc3\x7f\x06 \x05\xc3 \x12\x84\xd01\xf1\x82X\xcd\x04\x00\x0e\xf55\xcb\xd1\x8e\x0e\x1f\x00\x00\x00\x00IEND\xaeB`\x82",
|
||||
"mime_type": "image/jpeg",
|
||||
"type": "blob",
|
||||
},
|
||||
),
|
||||
},
|
||||
),
|
||||
"gen_ai.output.messages": (
|
||||
{
|
||||
"role": "model",
|
||||
"parts": ({"type": "text", "content": "This is a test."},),
|
||||
"parts": ({"content": "This is a test.", "type": "text"},),
|
||||
"finish_reason": "stop",
|
||||
},
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user