From dc517ba9db35b059e06f2a9c7cfd3a8ab0beeeb3 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Mon, 2 Mar 2026 09:53:14 +0100 Subject: [PATCH 01/11] test(anthropic): Stop mocking response iterator --- .../integrations/anthropic/test_anthropic.py | 699 ++++++++++-------- 1 file changed, 398 insertions(+), 301 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 4361ba9629..8688c326a6 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -1,6 +1,7 @@ import pytest from unittest import mock import json +import httpx try: from unittest.mock import AsyncMock @@ -12,7 +13,7 @@ async def __call__(self, *args, **kwargs): from anthropic import Anthropic, AnthropicError, AsyncAnthropic, AsyncStream, Stream -from anthropic.types import MessageDeltaUsage, TextDelta, Usage +from anthropic.types import MessageDeltaUsage, TextDelta, Usage, MessageStreamEvent from anthropic.types.content_block_delta_event import ContentBlockDeltaEvent from anthropic.types.content_block_start_event import ContentBlockStartEvent from anthropic.types.content_block_stop_event import ContentBlockStopEvent @@ -67,6 +68,13 @@ async def __call__(self, *args, **kwargs): ) +def sse_chunks(events): + for event in events: + payload = event.model_dump() + chunk = f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n" + yield chunk.encode("utf-8") + + async def async_iterator(values): for value in values: yield value @@ -224,39 +232,49 @@ def test_streaming_create_message( sentry_init, capture_events, send_default_pii, include_prompts ): client = Anthropic(api_key="z") - returned_stream = Stream(cast_to=None, response=None, client=client) - returned_stream._iterator = [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", + + response = httpx.Response( + 200, + content=b"".join( + sse_chunks( + [ + MessageStartEvent( + message=EXAMPLE_MESSAGE, + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text=" I'm Claude!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + ) ), - ] + ) + returned_stream = Stream( + cast_to=MessageStreamEvent, response=response, client=client + ) sentry_init( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -327,40 +345,48 @@ async def test_streaming_create_message_async( sentry_init, capture_events, send_default_pii, include_prompts ): client = AsyncAnthropic(api_key="z") - returned_stream = AsyncStream(cast_to=None, response=None, client=client) - returned_stream._iterator = async_iterator( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + + response = httpx.Response( + 200, + content=b"".join( + sse_chunks( + [ + MessageStartEvent( + message=EXAMPLE_MESSAGE, + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text=" I'm Claude!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + ) + ), + ) + returned_stream = AsyncStream( + cast_to=MessageStreamEvent, response=response, client=client ) sentry_init( @@ -435,65 +461,85 @@ def test_streaming_create_message_with_input_json_delta( sentry_init, capture_events, send_default_pii, include_prompts ): client = Anthropic(api_key="z") - returned_stream = Stream(cast_to=None, response=None, client=client) - returned_stream._iterator = [ - MessageStartEvent( - message=Message( - id="msg_0", - content=[], - model="claude-3-5-sonnet-20240620", - role="assistant", - stop_reason=None, - stop_sequence=None, - type="message", - usage=Usage(input_tokens=366, output_tokens=10), - ), - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=ToolUseBlock( - id="toolu_0", input={}, name="get_weather", type="tool_use" - ), - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta(partial_json="", type="input_json_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta(partial_json="{'location':", type="input_json_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta(partial_json=" 'S", type="input_json_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta(partial_json="an ", type="input_json_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta(partial_json="Francisco, C", type="input_json_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta(partial_json="A'}", type="input_json_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="tool_use", stop_sequence=None), - usage=MessageDeltaUsage(output_tokens=41), - type="message_delta", + + response = httpx.Response( + 200, + content=b"".join( + sse_chunks( + [ + MessageStartEvent( + message=Message( + id="msg_0", + content=[], + model="claude-3-5-sonnet-20240620", + role="assistant", + stop_reason=None, + stop_sequence=None, + type="message", + usage=Usage(input_tokens=366, output_tokens=10), + ), + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=ToolUseBlock( + id="toolu_0", input={}, name="get_weather", type="tool_use" + ), + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta(partial_json="", type="input_json_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta( + partial_json="{'location':", type="input_json_delta" + ), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta( + partial_json=" 'S", type="input_json_delta" + ), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta( + partial_json="an ", type="input_json_delta" + ), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta( + partial_json="Francisco, C", type="input_json_delta" + ), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta( + partial_json="A'}", type="input_json_delta" + ), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(stop_reason="tool_use", stop_sequence=None), + usage=MessageDeltaUsage(output_tokens=41), + type="message_delta", + ), + ] + ) ), - ] + ) + returned_stream = Stream( + cast_to=MessageStreamEvent, response=response, client=client + ) sentry_init( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -570,70 +616,83 @@ async def test_streaming_create_message_with_input_json_delta_async( sentry_init, capture_events, send_default_pii, include_prompts ): client = AsyncAnthropic(api_key="z") - returned_stream = AsyncStream(cast_to=None, response=None, client=client) - returned_stream._iterator = async_iterator( - [ - MessageStartEvent( - message=Message( - id="msg_0", - content=[], - model="claude-3-5-sonnet-20240620", - role="assistant", - stop_reason=None, - stop_sequence=None, - type="message", - usage=Usage(input_tokens=366, output_tokens=10), - ), - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=ToolUseBlock( - id="toolu_0", input={}, name="get_weather", type="tool_use" - ), - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta(partial_json="", type="input_json_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta( - partial_json="{'location':", type="input_json_delta" - ), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta(partial_json=" 'S", type="input_json_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta(partial_json="an ", type="input_json_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta( - partial_json="Francisco, C", type="input_json_delta" - ), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta(partial_json="A'}", type="input_json_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="tool_use", stop_sequence=None), - usage=MessageDeltaUsage(output_tokens=41), - type="message_delta", - ), - ] + response = httpx.Response( + 200, + content=b"".join( + sse_chunks( + [ + MessageStartEvent( + message=Message( + id="msg_0", + content=[], + model="claude-3-5-sonnet-20240620", + role="assistant", + stop_reason=None, + stop_sequence=None, + type="message", + usage=Usage(input_tokens=366, output_tokens=10), + ), + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=ToolUseBlock( + id="toolu_0", input={}, name="get_weather", type="tool_use" + ), + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta(partial_json="", type="input_json_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta( + partial_json="{'location':", type="input_json_delta" + ), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta( + partial_json=" 'S", type="input_json_delta" + ), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta( + partial_json="an ", type="input_json_delta" + ), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta( + partial_json="Francisco, C", type="input_json_delta" + ), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta( + partial_json="A'}", type="input_json_delta" + ), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(stop_reason="tool_use", stop_sequence=None), + usage=MessageDeltaUsage(output_tokens=41), + type="message_delta", + ), + ] + ) + ), + ) + returned_stream = AsyncStream( + cast_to=MessageStreamEvent, response=response, client=client ) sentry_init( @@ -1247,39 +1306,49 @@ def test_streaming_create_message_with_system_prompt( ): """Test that system prompts are properly captured in streaming mode.""" client = Anthropic(api_key="z") - returned_stream = Stream(cast_to=None, response=None, client=client) - returned_stream._iterator = [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", + + response = httpx.Response( + 200, + content=b"".join( + sse_chunks( + [ + MessageStartEvent( + message=EXAMPLE_MESSAGE, + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text=" I'm Claude!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + ) ), - ] + ) + returned_stream = Stream( + cast_to=MessageStreamEvent, response=response, client=client + ) sentry_init( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -1365,40 +1434,48 @@ async def test_streaming_create_message_with_system_prompt_async( ): """Test that system prompts are properly captured in streaming mode (async).""" client = AsyncAnthropic(api_key="z") - returned_stream = AsyncStream(cast_to=None, response=None, client=client) - returned_stream._iterator = async_iterator( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + + response = httpx.Response( + 200, + content=b"".join( + sse_chunks( + [ + MessageStartEvent( + message=EXAMPLE_MESSAGE, + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text=" I'm Claude!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + ) + ), + ) + returned_stream = AsyncStream( + cast_to=MessageStreamEvent, response=response, client=client ) sentry_init( @@ -2372,30 +2449,40 @@ def test_input_tokens_include_cache_read_streaming(sentry_init, capture_events): Same cache-hit scenario as non-streaming, using realistic streaming events. """ client = Anthropic(api_key="z") - returned_stream = Stream(cast_to=None, response=None, client=client) - returned_stream._iterator = [ - MessageStartEvent( - type="message_start", - message=Message( - id="id", - model="claude-sonnet-4-20250514", - role="assistant", - content=[], - type="message", - usage=Usage( - input_tokens=19, - output_tokens=0, - cache_read_input_tokens=2846, - cache_creation_input_tokens=0, - ), - ), - ), - MessageDeltaEvent( - type="message_delta", - delta=Delta(stop_reason="end_turn"), - usage=MessageDeltaUsage(output_tokens=14), + + response = httpx.Response( + 200, + content=b"".join( + sse_chunks( + [ + MessageStartEvent( + type="message_start", + message=Message( + id="id", + model="claude-sonnet-4-20250514", + role="assistant", + content=[], + type="message", + usage=Usage( + input_tokens=19, + output_tokens=0, + cache_read_input_tokens=2846, + cache_creation_input_tokens=0, + ), + ), + ), + MessageDeltaEvent( + type="message_delta", + delta=Delta(stop_reason="end_turn"), + usage=MessageDeltaUsage(output_tokens=14), + ), + ] + ) ), - ] + ) + returned_stream = Stream( + cast_to=MessageStreamEvent, response=response, client=client + ) sentry_init(integrations=[AnthropicIntegration()], traces_sample_rate=1.0) events = capture_events() @@ -2460,30 +2547,40 @@ def test_input_tokens_unchanged_without_caching(sentry_init, capture_events): def test_cache_tokens_streaming(sentry_init, capture_events): """Test cache tokens are tracked for streaming responses.""" client = Anthropic(api_key="z") - returned_stream = Stream(cast_to=None, response=None, client=client) - returned_stream._iterator = [ - MessageStartEvent( - type="message_start", - message=Message( - id="id", - model="claude-3-5-sonnet-20241022", - role="assistant", - content=[], - type="message", - usage=Usage( - input_tokens=100, - output_tokens=0, - cache_read_input_tokens=80, - cache_creation_input_tokens=20, - ), - ), - ), - MessageDeltaEvent( - type="message_delta", - delta=Delta(stop_reason="end_turn"), - usage=MessageDeltaUsage(output_tokens=10), + + response = httpx.Response( + 200, + content=b"".join( + sse_chunks( + [ + MessageStartEvent( + type="message_start", + message=Message( + id="id", + model="claude-3-5-sonnet-20241022", + role="assistant", + content=[], + type="message", + usage=Usage( + input_tokens=100, + output_tokens=0, + cache_read_input_tokens=80, + cache_creation_input_tokens=20, + ), + ), + ), + MessageDeltaEvent( + type="message_delta", + delta=Delta(stop_reason="end_turn"), + usage=MessageDeltaUsage(output_tokens=10), + ), + ] + ) ), - ] + ) + returned_stream = Stream( + cast_to=MessageStreamEvent, response=response, client=client + ) sentry_init(integrations=[AnthropicIntegration()], traces_sample_rate=1.0) events = capture_events() From 59c6a3840d8be29eacd5bf06a9707e9813fccc79 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 10 Mar 2026 13:08:18 +0100 Subject: [PATCH 02/11] . --- tests/conftest.py | 11 +++++++++++ .../openai_agents/test_openai_agents.py | 13 ++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0853013dfd..cb55183583 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1013,6 +1013,17 @@ async def inner(values): return inner +@pytest.fixture +def server_side_event_chunks(): + def inner(events): + for event in events: + payload = event.model_dump() + chunk = f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n" + yield chunk.encode("utf-8") + + return inner + + class MockServerRequestHandler(BaseHTTPRequestHandler): def do_GET(self): # noqa: N802 # Process an HTTP GET request and return a response. diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 117755b963..2ae7930013 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -1386,16 +1386,9 @@ def simple_test_tool(message: str) -> str: assert ai_client_span2["data"]["gen_ai.usage.total_tokens"] == 25 -def server_side_event_chunks(events): - for event in events: - payload = event.model_dump() - chunk = f"event: {payload['type']}\ndata: {json.dumps(payload)}\n\n" - yield chunk.encode("utf-8") - - @pytest.mark.asyncio async def test_hosted_mcp_tool_propagation_header_streamed( - sentry_init, test_agent, async_iterator + sentry_init, test_agent, async_iterator, server_side_event_chunks ): """ Test responses API is given trace propagation headers with HostedMCPTool. @@ -3162,7 +3155,9 @@ async def test_streaming_span_update_captures_response_data( @pytest.mark.asyncio -async def test_streaming_ttft_on_chat_span(sentry_init, test_agent, async_iterator): +async def test_streaming_ttft_on_chat_span( + sentry_init, test_agent, async_iterator, server_side_event_chunks +): """ Test that time-to-first-token (TTFT) is recorded on chat spans during streaming. From 370a3ce8a08970bfd4cf17c7df316790f4a4b10f Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 11 Mar 2026 16:36:47 +0100 Subject: [PATCH 03/11] . --- .../integrations/anthropic/test_anthropic.py | 614 +++++++++--------- 1 file changed, 300 insertions(+), 314 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index ee6cf84481..a412c33ecb 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -222,51 +222,46 @@ def test_streaming_create_message( send_default_pii, include_prompts, server_side_event_chunks, + get_model_response, + select_transactions_with_ai_client_spans, ): client = Anthropic(api_key="z") - response = httpx.Response( - 200, - content=b"".join( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] - ) - ), - ) - - returned_stream = Stream( - cast_to=MessageStreamEvent, response=response, client=client + response = get_model_response( + server_side_event_chunks( + [ + MessageStartEvent( + message=EXAMPLE_MESSAGE, + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text=" I'm Claude!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + ) ) sentry_init( @@ -275,7 +270,6 @@ def test_streaming_create_message( send_default_pii=send_default_pii, ) events = capture_events() - client.messages._post = mock.Mock(return_value=returned_stream) messages = [ { @@ -284,23 +278,27 @@ def test_streaming_create_message( } ] - with start_transaction(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _: + with start_transaction(name="anthropic"): + message = client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True + ) - for _ in message: - pass + for _ in message: + pass - assert message == returned_stream + events = select_transactions_with_ai_client_spans(events, "chat") assert len(events) == 1 (event,) = events assert event["type"] == "transaction" assert event["transaction"] == "anthropic" - assert len(event["spans"]) == 1 - (span,) = event["spans"] + span = next(span for span in event["spans"] if span["op"] == OP.GEN_AI_CHAT) assert span["op"] == OP.GEN_AI_CHAT assert span["description"] == "chat model" @@ -341,12 +339,12 @@ async def test_streaming_create_message_async( include_prompts, async_iterator, server_side_event_chunks, + get_model_response, ): client = AsyncAnthropic(api_key="z") - response = httpx.Response( - 200, - content=async_iterator( + response = get_model_response( + async_iterator( server_side_event_chunks( [ MessageStartEvent( @@ -383,9 +381,6 @@ async def test_streaming_create_message_async( ) ), ) - returned_stream = AsyncStream( - cast_to=MessageStreamEvent, response=response, client=client - ) sentry_init( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -393,7 +388,6 @@ async def test_streaming_create_message_async( send_default_pii=send_default_pii, ) events = capture_events() - client.messages._post = AsyncMock(return_value=returned_stream) messages = [ { @@ -402,15 +396,19 @@ async def test_streaming_create_message_async( } ] - with start_transaction(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _: + with start_transaction(name="anthropic"): + message = await client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True + ) - async for _ in message: - pass + async for _ in message: + pass - assert message == returned_stream assert len(events) == 1 (event,) = events @@ -461,86 +459,75 @@ def test_streaming_create_message_with_input_json_delta( send_default_pii, include_prompts, server_side_event_chunks, + get_model_response, ): client = Anthropic(api_key="z") - response = httpx.Response( - 200, - content=b"".join( - server_side_event_chunks( - [ - MessageStartEvent( - message=Message( - id="msg_0", - content=[], - model="claude-3-5-sonnet-20240620", - role="assistant", - stop_reason=None, - stop_sequence=None, - type="message", - usage=Usage(input_tokens=366, output_tokens=10), - ), - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=ToolUseBlock( - id="toolu_0", input={}, name="get_weather", type="tool_use" - ), - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta(partial_json="", type="input_json_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta( - partial_json="{'location':", type="input_json_delta" - ), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta( - partial_json=" 'S", type="input_json_delta" - ), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=InputJSONDelta( - partial_json="an ", type="input_json_delta" - ), - index=0, - type="content_block_delta", + response = get_model_response( + server_side_event_chunks( + [ + MessageStartEvent( + message=Message( + id="msg_0", + content=[], + model="claude-3-5-sonnet-20240620", + role="assistant", + stop_reason=None, + stop_sequence=None, + type="message", + usage=Usage(input_tokens=366, output_tokens=10), ), - ContentBlockDeltaEvent( - delta=InputJSONDelta( - partial_json="Francisco, C", type="input_json_delta" - ), - index=0, - type="content_block_delta", + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=ToolUseBlock( + id="toolu_0", input={}, name="get_weather", type="tool_use" ), - ContentBlockDeltaEvent( - delta=InputJSONDelta( - partial_json="A'}", type="input_json_delta" - ), - index=0, - type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta(partial_json="", type="input_json_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta( + partial_json="{'location':", type="input_json_delta" ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="tool_use", stop_sequence=None), - usage=MessageDeltaUsage(output_tokens=41), - type="message_delta", + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta(partial_json=" 'S", type="input_json_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta(partial_json="an ", type="input_json_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta( + partial_json="Francisco, C", type="input_json_delta" ), - ] - ) - ), - ) - returned_stream = Stream( - cast_to=MessageStreamEvent, response=response, client=client + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta(partial_json="A'}", type="input_json_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(stop_reason="tool_use", stop_sequence=None), + usage=MessageDeltaUsage(output_tokens=41), + type="message_delta", + ), + ] + ) ) sentry_init( @@ -549,7 +536,6 @@ def test_streaming_create_message_with_input_json_delta( send_default_pii=send_default_pii, ) events = capture_events() - client.messages._post = mock.Mock(return_value=returned_stream) messages = [ { @@ -558,15 +544,19 @@ def test_streaming_create_message_with_input_json_delta( } ] - with start_transaction(name="anthropic"): - message = client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _: + with start_transaction(name="anthropic"): + message = client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True + ) - for _ in message: - pass + for _ in message: + pass - assert message == returned_stream assert len(events) == 1 (event,) = events @@ -621,11 +611,11 @@ async def test_streaming_create_message_with_input_json_delta_async( include_prompts, async_iterator, server_side_event_chunks, + get_model_response, ): client = AsyncAnthropic(api_key="z") - response = httpx.Response( - 200, - content=b"".join( + response = get_model_response( + async_iterator( server_side_event_chunks( [ MessageStartEvent( @@ -696,10 +686,7 @@ async def test_streaming_create_message_with_input_json_delta_async( ), ] ) - ), - ) - returned_stream = AsyncStream( - cast_to=MessageStreamEvent, response=response, client=client + ) ) sentry_init( @@ -708,7 +695,6 @@ async def test_streaming_create_message_with_input_json_delta_async( send_default_pii=send_default_pii, ) events = capture_events() - client.messages._post = AsyncMock(return_value=returned_stream) messages = [ { @@ -717,15 +703,19 @@ async def test_streaming_create_message_with_input_json_delta_async( } ] - with start_transaction(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, messages=messages, model="model", stream=True - ) + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _: + with start_transaction(name="anthropic"): + message = await client.messages.create( + max_tokens=1024, messages=messages, model="model", stream=True + ) - async for _ in message: - pass + async for _ in message: + pass - assert message == returned_stream assert len(events) == 1 (event,) = events @@ -1314,51 +1304,46 @@ def test_streaming_create_message_with_system_prompt( send_default_pii, include_prompts, server_side_event_chunks, + get_model_response, ): """Test that system prompts are properly captured in streaming mode.""" client = Anthropic(api_key="z") - response = httpx.Response( - 200, - content=b"".join( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] - ) - ), - ) - returned_stream = Stream( - cast_to=MessageStreamEvent, response=response, client=client + response = get_model_response( + server_side_event_chunks( + [ + MessageStartEvent( + message=EXAMPLE_MESSAGE, + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text=" I'm Claude!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + ) ) sentry_init( @@ -1367,7 +1352,6 @@ def test_streaming_create_message_with_system_prompt( send_default_pii=send_default_pii, ) events = capture_events() - client.messages._post = mock.Mock(return_value=returned_stream) messages = [ { @@ -1376,19 +1360,23 @@ def test_streaming_create_message_with_system_prompt( } ] - with start_transaction(name="anthropic"): - message = client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - stream=True, - system="You are a helpful assistant.", - ) + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _: + with start_transaction(name="anthropic"): + message = client.messages.create( + max_tokens=1024, + messages=messages, + model="model", + stream=True, + system="You are a helpful assistant.", + ) - for _ in message: - pass + for _ in message: + pass - assert message == returned_stream assert len(events) == 1 (event,) = events @@ -1447,13 +1435,13 @@ async def test_streaming_create_message_with_system_prompt_async( include_prompts, async_iterator, server_side_event_chunks, + get_model_response, ): """Test that system prompts are properly captured in streaming mode (async).""" client = AsyncAnthropic(api_key="z") - response = httpx.Response( - 200, - content=b"".join( + response = get_model_response( + async_iterator( server_side_event_chunks( [ MessageStartEvent( @@ -1488,10 +1476,7 @@ async def test_streaming_create_message_with_system_prompt_async( ), ] ) - ), - ) - returned_stream = AsyncStream( - cast_to=MessageStreamEvent, response=response, client=client + ) ) sentry_init( @@ -1500,7 +1485,6 @@ async def test_streaming_create_message_with_system_prompt_async( send_default_pii=send_default_pii, ) events = capture_events() - client.messages._post = AsyncMock(return_value=returned_stream) messages = [ { @@ -1509,19 +1493,23 @@ async def test_streaming_create_message_with_system_prompt_async( } ] - with start_transaction(name="anthropic"): - message = await client.messages.create( - max_tokens=1024, - messages=messages, - model="model", - stream=True, - system="You are a helpful assistant.", - ) + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _: + with start_transaction(name="anthropic"): + message = await client.messages.create( + max_tokens=1024, + messages=messages, + model="model", + stream=True, + system="You are a helpful assistant.", + ) - async for _ in message: - pass + async for _ in message: + pass - assert message == returned_stream assert len(events) == 1 (event,) = events @@ -2459,7 +2447,7 @@ def test_input_tokens_include_cache_read_nonstreaming(sentry_init, capture_event def test_input_tokens_include_cache_read_streaming( - sentry_init, capture_events, server_side_event_chunks + sentry_init, capture_events, server_side_event_chunks, get_model_response ): """ Test that gen_ai.usage.input_tokens includes cache_read tokens (streaming). @@ -2468,52 +2456,50 @@ def test_input_tokens_include_cache_read_streaming( """ client = Anthropic(api_key="z") - response = httpx.Response( - 200, - content=b"".join( - server_side_event_chunks( - [ - MessageStartEvent( - type="message_start", - message=Message( - id="id", - model="claude-sonnet-4-20250514", - role="assistant", - content=[], - type="message", - usage=Usage( - input_tokens=19, - output_tokens=0, - cache_read_input_tokens=2846, - cache_creation_input_tokens=0, - ), + response = get_model_response( + server_side_event_chunks( + [ + MessageStartEvent( + type="message_start", + message=Message( + id="id", + model="claude-sonnet-4-20250514", + role="assistant", + content=[], + type="message", + usage=Usage( + input_tokens=19, + output_tokens=0, + cache_read_input_tokens=2846, + cache_creation_input_tokens=0, ), ), - MessageDeltaEvent( - type="message_delta", - delta=Delta(stop_reason="end_turn"), - usage=MessageDeltaUsage(output_tokens=14), - ), - ] - ) - ), - ) - returned_stream = Stream( - cast_to=MessageStreamEvent, response=response, client=client + ), + MessageDeltaEvent( + type="message_delta", + delta=Delta(stop_reason="end_turn"), + usage=MessageDeltaUsage(output_tokens=14), + ), + ] + ) ) sentry_init(integrations=[AnthropicIntegration()], traces_sample_rate=1.0) events = capture_events() - client.messages._post = mock.Mock(return_value=returned_stream) - with start_transaction(name="anthropic"): - for _ in client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "What is 5+5?"}], - model="claude-sonnet-4-20250514", - stream=True, - ): - pass + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _: + with start_transaction(name="anthropic"): + for _ in client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "What is 5+5?"}], + model="claude-sonnet-4-20250514", + stream=True, + ): + pass (span,) = events[0]["spans"] @@ -2562,56 +2548,56 @@ def test_input_tokens_unchanged_without_caching(sentry_init, capture_events): assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 32 # 20 + 12 -def test_cache_tokens_streaming(sentry_init, capture_events, server_side_event_chunks): +def test_cache_tokens_streaming( + sentry_init, capture_events, server_side_event_chunks, get_model_response +): """Test cache tokens are tracked for streaming responses.""" client = Anthropic(api_key="z") - response = httpx.Response( - 200, - content=b"".join( - server_side_event_chunks( - [ - MessageStartEvent( - type="message_start", - message=Message( - id="id", - model="claude-3-5-sonnet-20241022", - role="assistant", - content=[], - type="message", - usage=Usage( - input_tokens=100, - output_tokens=0, - cache_read_input_tokens=80, - cache_creation_input_tokens=20, - ), + response = get_model_response( + server_side_event_chunks( + [ + MessageStartEvent( + type="message_start", + message=Message( + id="id", + model="claude-3-5-sonnet-20241022", + role="assistant", + content=[], + type="message", + usage=Usage( + input_tokens=100, + output_tokens=0, + cache_read_input_tokens=80, + cache_creation_input_tokens=20, ), ), - MessageDeltaEvent( - type="message_delta", - delta=Delta(stop_reason="end_turn"), - usage=MessageDeltaUsage(output_tokens=10), - ), - ] - ) - ), - ) - returned_stream = Stream( - cast_to=MessageStreamEvent, response=response, client=client + ), + MessageDeltaEvent( + type="message_delta", + delta=Delta(stop_reason="end_turn"), + usage=MessageDeltaUsage(output_tokens=10), + ), + ] + ) ) sentry_init(integrations=[AnthropicIntegration()], traces_sample_rate=1.0) events = capture_events() - client.messages._post = mock.Mock(return_value=returned_stream) - with start_transaction(name="anthropic"): - for _ in client.messages.create( - max_tokens=1024, - messages=[{"role": "user", "content": "Hello"}], - model="claude-3-5-sonnet-20241022", - stream=True, - ): - pass + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _: + with start_transaction(name="anthropic"): + for _ in client.messages.create( + max_tokens=1024, + messages=[{"role": "user", "content": "Hello"}], + model="claude-3-5-sonnet-20241022", + stream=True, + ): + pass (span,) = events[0]["spans"] # input_tokens normalized: 100 + 80 (cache_read) + 20 (cache_write) = 200 From 09516b79ed3aab92df15ad33106c26e425665409 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 11 Mar 2026 16:37:55 +0100 Subject: [PATCH 04/11] . --- tests/integrations/anthropic/test_anthropic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index a412c33ecb..2128a2d4a6 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -1,7 +1,6 @@ import pytest from unittest import mock import json -import httpx try: from unittest.mock import AsyncMock From 020f1de94f9e1502e3c5762a0fbe1b28a9431b25 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 11 Mar 2026 16:45:00 +0100 Subject: [PATCH 05/11] . --- tests/conftest.py | 51 ++++++++++++++----- .../integrations/anthropic/test_anthropic.py | 16 ++++++ 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index cb55183583..7f76fc2aee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,13 +57,6 @@ from collections.abc import Iterator try: - from anyio import create_memory_object_stream, create_task_group, EndOfStream - from mcp.types import ( - JSONRPCMessage, - JSONRPCNotification, - JSONRPCRequest, - ) - from mcp.shared.message import SessionMessage from httpx import ( ASGITransport, Request as HttpxRequest, @@ -71,6 +64,22 @@ AsyncByteStream, AsyncClient, ) +except ImportError: + ASGITransport = None + HttpxRequest = None + HttpxResponse = None + AsyncByteStream = None + AsyncClient = None + + +try: + from anyio import create_memory_object_stream, create_task_group, EndOfStream + from mcp.types import ( + JSONRPCMessage, + JSONRPCNotification, + JSONRPCRequest, + ) + from mcp.shared.message import SessionMessage except ImportError: create_memory_object_stream = None create_task_group = None @@ -81,12 +90,6 @@ JSONRPCRequest = None SessionMessage = None - ASGITransport = None - HttpxRequest = None - HttpxResponse = None - AsyncByteStream = None - AsyncClient = None - SENTRY_EVENT_SCHEMA = "./checkouts/data-schemas/relay/event.schema.json" @@ -1024,6 +1027,28 @@ def inner(events): return inner +@pytest.fixture +def get_model_response(): + def inner(response_content, serialize_pydantic=False): + model_request = HttpxRequest( + "POST", + "/responses", + ) + + if serialize_pydantic: + response_content = json.dumps(response_content.model_dump()).encode("utf-8") + + response = HttpxResponse( + 200, + request=model_request, + content=response_content, + ) + + return response + + return inner + + class MockServerRequestHandler(BaseHTTPRequestHandler): def do_GET(self): # noqa: N802 # Process an HTTP GET request and return a response. diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 2128a2d4a6..d0b682e52d 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -67,6 +67,22 @@ async def __call__(self, *args, **kwargs): ) +@pytest.fixture() +def select_transactions_with_ai_client_spans(): + def inner(events, operation_name): + return [ + transaction + for transaction in events + if transaction["type"] == "transaction" + and any( + span["data"].get("gen_ai.operation.name") == operation_name + for span in transaction.get("spans", []) + ) + ] + + return inner + + @pytest.mark.parametrize( "send_default_pii, include_prompts", [ From 2a34bd4cbceec49ed44e836e48dbcdd7f1cc4743 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 11 Mar 2026 17:26:41 +0100 Subject: [PATCH 06/11] . --- .../openai_agents/test_openai_agents.py | 107 +++++++++--------- 1 file changed, 56 insertions(+), 51 deletions(-) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 2ae7930013..e1d0b46cef 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -201,25 +201,6 @@ def test_agent_custom_model(): ) -@pytest.fixture -def get_model_response(): - def inner(response_content): - model_request = httpx.Request( - "POST", - "/responses", - ) - - response = httpx.Response( - 200, - request=model_request, - content=json.dumps(response_content.model_dump()).encode("utf-8"), - ) - - return response - - return inner - - @pytest.mark.asyncio async def test_agent_invocation_span_no_pii( sentry_init, capture_events, test_agent, mock_model_response, get_model_response @@ -228,7 +209,7 @@ async def test_agent_invocation_span_no_pii( model = OpenAIResponsesModel(model="gpt-4", openai_client=client) agent = test_agent.clone(model=model) - response = get_model_response(mock_model_response) + response = get_model_response(mock_model_response, serialize_pydantic=True) with patch.object( agent.model._client._client, @@ -377,7 +358,7 @@ async def test_agent_invocation_span( model = OpenAIResponsesModel(model="gpt-4", openai_client=client) agent = test_agent_with_instructions(instructions).clone(model=model) - response = get_model_response(mock_model_response) + response = get_model_response(mock_model_response, serialize_pydantic=True) with patch.object( agent.model._client._client, @@ -538,7 +519,7 @@ async def test_client_span_custom_model( model = OpenAIResponsesModel(model="my-custom-model", openai_client=client) agent = test_agent_custom_model.clone(model=model) - response = get_model_response(mock_model_response) + response = get_model_response(mock_model_response, serialize_pydantic=True) with patch.object( agent.model._client._client, @@ -581,7 +562,7 @@ def test_agent_invocation_span_sync_no_pii( model = OpenAIResponsesModel(model="gpt-4", openai_client=client) agent = test_agent.clone(model=model) - response = get_model_response(mock_model_response) + response = get_model_response(mock_model_response, serialize_pydantic=True) with patch.object( agent.model._client._client, @@ -724,7 +705,7 @@ def test_agent_invocation_span_sync( model = OpenAIResponsesModel(model="gpt-4", openai_client=client) agent = test_agent_with_instructions(instructions).clone(model=model) - response = get_model_response(mock_model_response) + response = get_model_response(mock_model_response, serialize_pydantic=True) with patch.object( agent.model._client._client, @@ -906,7 +887,8 @@ async def test_handoff_span(sentry_init, capture_events, get_model_response): ), total_tokens=30, ), - ) + ), + serialize_pydantic=True, ) final_response = get_model_response( @@ -944,7 +926,8 @@ async def test_handoff_span(sentry_init, capture_events, get_model_response): ), total_tokens=30, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -1032,7 +1015,8 @@ async def test_max_turns_before_handoff_span( ), total_tokens=30, ), - ) + ), + serialize_pydantic=True, ) final_response = get_model_response( @@ -1070,7 +1054,8 @@ async def test_max_turns_before_handoff_span( ), total_tokens=30, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -1152,7 +1137,8 @@ def simple_test_tool(message: str) -> str: ), total_tokens=15, ), - ) + ), + serialize_pydantic=True, ) final_response = get_model_response( @@ -1190,7 +1176,8 @@ def simple_test_tool(message: str) -> str: ), total_tokens=25, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -1898,7 +1885,8 @@ async def test_mcp_tool_execution_spans( ), total_tokens=15, ), - ) + ), + serialize_pydantic=True, ) final_response = get_model_response( @@ -1936,7 +1924,8 @@ async def test_mcp_tool_execution_spans( ), total_tokens=25, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -2028,7 +2017,8 @@ async def test_mcp_tool_execution_with_error( ), total_tokens=15, ), - ) + ), + serialize_pydantic=True, ) final_response = get_model_response( @@ -2066,7 +2056,8 @@ async def test_mcp_tool_execution_with_error( ), total_tokens=25, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -2156,7 +2147,8 @@ async def test_mcp_tool_execution_without_pii( ), total_tokens=15, ), - ) + ), + serialize_pydantic=True, ) final_response = get_model_response( @@ -2194,7 +2186,8 @@ async def test_mcp_tool_execution_without_pii( ), total_tokens=25, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -2252,7 +2245,7 @@ async def test_multiple_agents_asyncio( model = OpenAIResponsesModel(model="gpt-4", openai_client=client) agent = test_agent.clone(model=model) - response = get_model_response(mock_model_response) + response = get_model_response(mock_model_response, serialize_pydantic=True) with patch.object( agent.model._client._client, @@ -2382,7 +2375,8 @@ def failing_tool(message: str) -> str: ), total_tokens=15, ), - ) + ), + serialize_pydantic=True, ) final_response = get_model_response( @@ -2420,7 +2414,8 @@ def failing_tool(message: str) -> str: ), total_tokens=25, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -2515,7 +2510,8 @@ async def test_invoke_agent_span_includes_usage_data( ), total_tokens=30, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -2606,7 +2602,8 @@ async def test_ai_client_span_includes_response_model( ), total_tokens=30, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -2693,7 +2690,8 @@ async def test_ai_client_span_response_model_with_chat_completions( ), total_tokens=40, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -2770,7 +2768,8 @@ def calculator(a: int, b: int) -> int: ), total_tokens=15, ), - ) + ), + serialize_pydantic=True, ) final_response = get_model_response( @@ -2808,7 +2807,8 @@ def calculator(a: int, b: int) -> int: ), total_tokens=35, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -2896,7 +2896,8 @@ async def test_invoke_agent_span_includes_response_model( ), total_tokens=30, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -2985,7 +2986,8 @@ def calculator(a: int, b: int) -> int: ), total_tokens=15, ), - ) + ), + serialize_pydantic=True, ) second_response = get_model_response( @@ -3023,7 +3025,8 @@ def calculator(a: int, b: int) -> int: ), total_tokens=35, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -3325,7 +3328,7 @@ async def test_conversation_id_on_all_spans( model = OpenAIResponsesModel(model="gpt-4", openai_client=client) agent = test_agent.clone(model=model) - response = get_model_response(mock_model_response) + response = get_model_response(mock_model_response, serialize_pydantic=True) with patch.object( agent.model._client._client, @@ -3418,7 +3421,8 @@ def simple_tool(message: str) -> str: ), total_tokens=15, ), - ) + ), + serialize_pydantic=True, ) final_response = get_model_response( @@ -3456,7 +3460,8 @@ def simple_tool(message: str) -> str: ), total_tokens=30, ), - ) + ), + serialize_pydantic=True, ) with patch.object( @@ -3519,7 +3524,7 @@ async def test_no_conversation_id_when_not_provided( model = OpenAIResponsesModel(model="gpt-4", openai_client=client) agent = test_agent.clone(model=model) - response = get_model_response(mock_model_response) + response = get_model_response(mock_model_response, serialize_pydantic=True) with patch.object( agent.model._client._client, From 340b49427e2dd61f259d133e2e41143a2307ccc5 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 11 Mar 2026 17:29:05 +0100 Subject: [PATCH 07/11] . --- tests/integrations/openai_agents/test_openai_agents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index e1d0b46cef..12736f6229 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -1574,7 +1574,7 @@ async def test_hosted_mcp_tool_propagation_headers( release="d08ebdb9309e1b004c6f52202de58a09c2268e42", ) - response = get_model_response(EXAMPLE_RESPONSE) + response = get_model_response(EXAMPLE_RESPONSE, serialize_pydantic=True) with patch.object( agent_with_tool.model._client._client, From d2e37fa04966a4cf520e4f30612e1fdd304d466a Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 11 Mar 2026 18:07:57 +0100 Subject: [PATCH 08/11] remove fixure --- tests/integrations/anthropic/test_anthropic.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index d0b682e52d..c069f6fa6a 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -67,22 +67,6 @@ async def __call__(self, *args, **kwargs): ) -@pytest.fixture() -def select_transactions_with_ai_client_spans(): - def inner(events, operation_name): - return [ - transaction - for transaction in events - if transaction["type"] == "transaction" - and any( - span["data"].get("gen_ai.operation.name") == operation_name - for span in transaction.get("spans", []) - ) - ] - - return inner - - @pytest.mark.parametrize( "send_default_pii, include_prompts", [ @@ -238,7 +222,6 @@ def test_streaming_create_message( include_prompts, server_side_event_chunks, get_model_response, - select_transactions_with_ai_client_spans, ): client = Anthropic(api_key="z") @@ -306,7 +289,6 @@ def test_streaming_create_message( for _ in message: pass - events = select_transactions_with_ai_client_spans(events, "chat") assert len(events) == 1 (event,) = events From 9d095a29b03351bb8c444f92e65816f54593af2b Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 11 Mar 2026 18:08:36 +0100 Subject: [PATCH 09/11] remove unused import --- tests/integrations/anthropic/test_anthropic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index c069f6fa6a..53589a9507 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -12,7 +12,7 @@ async def __call__(self, *args, **kwargs): from anthropic import Anthropic, AnthropicError, AsyncAnthropic, AsyncStream, Stream -from anthropic.types import MessageDeltaUsage, TextDelta, Usage, MessageStreamEvent +from anthropic.types import MessageDeltaUsage, TextDelta, Usage from anthropic.types.content_block_delta_event import ContentBlockDeltaEvent from anthropic.types.content_block_start_event import ContentBlockStartEvent from anthropic.types.content_block_stop_event import ContentBlockStopEvent From d79629142ddaa879023a166986a834d0c7112bbb Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 11 Mar 2026 18:10:23 +0100 Subject: [PATCH 10/11] . --- .../integrations/anthropic/test_anthropic.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 53589a9507..9847df1c57 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -220,8 +220,8 @@ def test_streaming_create_message( capture_events, send_default_pii, include_prompts, - server_side_event_chunks, get_model_response, + server_side_event_chunks, ): client = Anthropic(api_key="z") @@ -335,8 +335,8 @@ async def test_streaming_create_message_async( send_default_pii, include_prompts, async_iterator, - server_side_event_chunks, get_model_response, + server_side_event_chunks, ): client = AsyncAnthropic(api_key="z") @@ -455,8 +455,8 @@ def test_streaming_create_message_with_input_json_delta( capture_events, send_default_pii, include_prompts, - server_side_event_chunks, get_model_response, + server_side_event_chunks, ): client = Anthropic(api_key="z") @@ -607,8 +607,8 @@ async def test_streaming_create_message_with_input_json_delta_async( send_default_pii, include_prompts, async_iterator, - server_side_event_chunks, get_model_response, + server_side_event_chunks, ): client = AsyncAnthropic(api_key="z") response = get_model_response( @@ -1300,8 +1300,8 @@ def test_streaming_create_message_with_system_prompt( capture_events, send_default_pii, include_prompts, - server_side_event_chunks, get_model_response, + server_side_event_chunks, ): """Test that system prompts are properly captured in streaming mode.""" client = Anthropic(api_key="z") @@ -1431,8 +1431,8 @@ async def test_streaming_create_message_with_system_prompt_async( send_default_pii, include_prompts, async_iterator, - server_side_event_chunks, get_model_response, + server_side_event_chunks, ): """Test that system prompts are properly captured in streaming mode (async).""" client = AsyncAnthropic(api_key="z") @@ -2444,7 +2444,10 @@ def test_input_tokens_include_cache_read_nonstreaming(sentry_init, capture_event def test_input_tokens_include_cache_read_streaming( - sentry_init, capture_events, server_side_event_chunks, get_model_response + sentry_init, + capture_events, + get_model_response, + server_side_event_chunks, ): """ Test that gen_ai.usage.input_tokens includes cache_read tokens (streaming). @@ -2546,7 +2549,10 @@ def test_input_tokens_unchanged_without_caching(sentry_init, capture_events): def test_cache_tokens_streaming( - sentry_init, capture_events, server_side_event_chunks, get_model_response + sentry_init, + capture_events, + get_model_response, + server_side_event_chunks, ): """Test cache tokens are tracked for streaming responses.""" client = Anthropic(api_key="z") From 1edececf0658fe5823a95baa8ec8661be3794d2a Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Wed, 11 Mar 2026 18:36:30 +0100 Subject: [PATCH 11/11] order --- tests/integrations/anthropic/test_anthropic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 9847df1c57..c6a071b90f 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -334,8 +334,8 @@ async def test_streaming_create_message_async( capture_events, send_default_pii, include_prompts, - async_iterator, get_model_response, + async_iterator, server_side_event_chunks, ): client = AsyncAnthropic(api_key="z") @@ -606,8 +606,8 @@ async def test_streaming_create_message_with_input_json_delta_async( capture_events, send_default_pii, include_prompts, - async_iterator, get_model_response, + async_iterator, server_side_event_chunks, ): client = AsyncAnthropic(api_key="z") @@ -1430,8 +1430,8 @@ async def test_streaming_create_message_with_system_prompt_async( capture_events, send_default_pii, include_prompts, - async_iterator, get_model_response, + async_iterator, server_side_event_chunks, ): """Test that system prompts are properly captured in streaming mode (async)."""