diff --git a/docs/05_bring_your_own_model_provider.md b/docs/05_bring_your_own_model_provider.md index 66cb5013..04f17d48 100644 --- a/docs/05_bring_your_own_model_provider.md +++ b/docs/05_bring_your_own_model_provider.md @@ -135,6 +135,46 @@ class MyImageQAProvider(ImageQAProvider): ``` +### Execution Cost Tracking + +The built-in VLM providers include default pricing for supported models. You can override the pricing on any provider by passing `input_cost_per_million_tokens` and `output_cost_per_million_tokens`: + +```python +from askui import AgentSettings, ComputerAgent +from askui.model_providers import AnthropicVlmProvider +from askui.reporting import SimpleHtmlReporter + +with ComputerAgent( + reporters=[SimpleHtmlReporter()], + settings=AgentSettings( + vlm_provider=AnthropicVlmProvider( + model_id="claude-sonnet-4-6", + input_cost_per_million_tokens=3.0, + output_cost_per_million_tokens=15.0, + ), + ), +) as agent: + agent.act("Open settings") +``` + +If you implement a fully custom `VlmProvider`, override the `pricing` property to enable cost tracking: + +```python +from askui.model_providers import VlmProvider, ModelPricing + +class MyVlmProvider(VlmProvider): + @property + def pricing(self) -> ModelPricing | None: + return ModelPricing( + input_cost_per_million_tokens=1.0, + output_cost_per_million_tokens=5.0, + ) + + # ... rest of implementation +``` + +--- + ## Advanced: Injecting a Custom Client For full control over HTTP settings (timeouts, proxies, retries), you can inject a pre-configured client: diff --git a/docs/08_reporting.md b/docs/08_reporting.md index 10a43f32..6041fafd 100644 --- a/docs/08_reporting.md +++ b/docs/08_reporting.md @@ -32,6 +32,15 @@ This generates an HTML file (typically in the current directory) showing: SimpleHtmlReporter(output_dir="./execution_reports", filename="agent_run.html") ``` +### Execution Cost Tracking + +The HTML report automatically shows the estimated API cost when using a `VlmProvider` with pricing information. The built-in Anthropic and AskUI providers include default pricing for supported Claude models. + +The report will display: +- Total estimated cost +- Per-token rates used for the calculation +- Input and output token breakdowns (as before) + ### Custom Reporters Create custom reporters by implementing the `Reporter` interface: diff --git a/src/askui/agent_base.py b/src/askui/agent_base.py index 3f72d84c..5df3e1a2 100644 --- a/src/askui/agent_base.py +++ b/src/askui/agent_base.py @@ -75,7 +75,12 @@ def __init__( # Create conversation with speakers and model providers speakers = Speakers() _callbacks = list(callbacks or []) - _callbacks.append(UsageTrackingCallback(reporter=self._reporter)) + _callbacks.append( + UsageTrackingCallback( + reporter=self._reporter, + pricing=self._vlm_provider.pricing, + ) + ) self._conversation = Conversation( speakers=speakers, vlm_provider=self._vlm_provider, diff --git a/src/askui/model_providers/__init__.py b/src/askui/model_providers/__init__.py index a4f08f0d..add59506 100644 --- a/src/askui/model_providers/__init__.py +++ b/src/askui/model_providers/__init__.py @@ -23,6 +23,7 @@ from askui.model_providers.google_image_qa_provider import GoogleImageQAProvider from askui.model_providers.image_qa_provider import ImageQAProvider from askui.model_providers.vlm_provider import VlmProvider +from askui.utils.model_pricing import ModelPricing __all__ = [ "AnthropicImageQAProvider", @@ -33,5 +34,6 @@ "DetectionProvider", "GoogleImageQAProvider", "ImageQAProvider", + "ModelPricing", "VlmProvider", ] diff --git a/src/askui/model_providers/anthropic_vlm_provider.py b/src/askui/model_providers/anthropic_vlm_provider.py index b0e5d186..cc7525ec 100644 --- a/src/askui/model_providers/anthropic_vlm_provider.py +++ b/src/askui/model_providers/anthropic_vlm_provider.py @@ -16,6 +16,7 @@ ) from askui.models.shared.prompts import SystemPrompt from askui.models.shared.tools import ToolCollection +from askui.utils.model_pricing import ModelPricing _DEFAULT_MODEL_ID = "claude-sonnet-4-6" @@ -38,6 +39,11 @@ class AnthropicVlmProvider(VlmProvider): `\"claude-sonnet-4-6\"`. client (Anthropic | None, optional): Pre-configured Anthropic client. If provided, other connection parameters are ignored. + input_cost_per_million_tokens (float | None, optional): Override + cost in USD per 1M input tokens. Both cost params must be set + to override the built-in defaults. + output_cost_per_million_tokens (float | None, optional): Override + cost in USD per 1M output tokens. Example: ```python @@ -60,6 +66,8 @@ def __init__( auth_token: str | None = None, model_id: str | None = None, client: Anthropic | None = None, + input_cost_per_million_tokens: float | None = None, + output_cost_per_million_tokens: float | None = None, ) -> None: self._model_id_value = ( model_id or os.environ.get("VLM_PROVIDER_MODEL_ID") or _DEFAULT_MODEL_ID @@ -72,12 +80,22 @@ def __init__( base_url=base_url, auth_token=auth_token, ) + self._pricing = ModelPricing.for_model( + self._model_id_value, + input_cost_per_million_tokens=input_cost_per_million_tokens, + output_cost_per_million_tokens=output_cost_per_million_tokens, + ) @property @override def model_id(self) -> str: return self._model_id_value + @property + @override + def pricing(self) -> ModelPricing | None: + return self._pricing + @cached_property def _messages_api(self) -> AnthropicMessagesApi: """Lazily initialise the AnthropicMessagesApi on first use.""" diff --git a/src/askui/model_providers/askui_vlm_provider.py b/src/askui/model_providers/askui_vlm_provider.py index 5dfc9d29..d149deff 100644 --- a/src/askui/model_providers/askui_vlm_provider.py +++ b/src/askui/model_providers/askui_vlm_provider.py @@ -37,7 +37,6 @@ class AskUIVlmProvider(VlmProvider): `"claude-sonnet-4-6"`. client (Anthropic | None, optional): Pre-configured Anthropic client. If provided, `workspace_id` and `token` are ignored. - Example: ```python from askui import AgentSettings, ComputerAgent diff --git a/src/askui/model_providers/vlm_provider.py b/src/askui/model_providers/vlm_provider.py index fc1f046c..1e98b972 100644 --- a/src/askui/model_providers/vlm_provider.py +++ b/src/askui/model_providers/vlm_provider.py @@ -10,6 +10,7 @@ ) from askui.models.shared.prompts import SystemPrompt from askui.models.shared.tools import ToolCollection +from askui.utils.model_pricing import ModelPricing class VlmProvider(ABC): @@ -43,6 +44,15 @@ class VlmProvider(ABC): def model_id(self) -> str: """The model identifier used by this provider.""" + @property + def pricing(self) -> ModelPricing | None: + """Pricing information for this provider's model. + + Returns ``None`` if no pricing information is available. + Override in subclasses to provide model-specific pricing. + """ + return None + @abstractmethod def create_message( self, diff --git a/src/askui/models/shared/usage_tracking_callback.py b/src/askui/models/shared/usage_tracking_callback.py index 3bc58206..5e245434 100644 --- a/src/askui/models/shared/usage_tracking_callback.py +++ b/src/askui/models/shared/usage_tracking_callback.py @@ -5,15 +5,46 @@ from typing import TYPE_CHECKING from opentelemetry import trace +from pydantic import BaseModel from typing_extensions import override -from askui.models.shared.agent_message_param import UsageParam from askui.models.shared.conversation_callback import ConversationCallback -from askui.reporting import NULL_REPORTER, Reporter +from askui.reporting import NULL_REPORTER if TYPE_CHECKING: + from askui.models.shared.agent_message_param import UsageParam from askui.models.shared.conversation import Conversation + from askui.reporting import Reporter from askui.speaker.speaker import SpeakerResult + from askui.utils.model_pricing import ModelPricing + + +class UsageSummary(BaseModel): + """Accumulated token usage and optional cost breakdown for a conversation. + + Args: + input_tokens (int | None): Total input tokens sent to the API. + output_tokens (int | None): Total output tokens generated. + cache_creation_input_tokens (int | None): Tokens used for cache creation. + cache_read_input_tokens (int | None): Tokens read from cache. + input_cost (float | None): Computed input cost in `currency`. + output_cost (float | None): Computed output cost in `currency`. + total_cost (float | None): Sum of `input_cost` and `output_cost`. + currency (str | None): ISO 4217 currency code (e.g. ``"USD"``). + input_cost_per_million_tokens (float | None): Rate used to compute `input_cost`. + output_cost_per_million_tokens (float|None): Rate used to compute `output_cost`. + """ + + input_tokens: int | None = None + output_tokens: int | None = None + cache_creation_input_tokens: int | None = None + cache_read_input_tokens: int | None = None + input_cost: float | None = None + output_cost: float | None = None + total_cost: float | None = None + currency: str | None = None + input_cost_per_million_tokens: float | None = None + output_cost_per_million_tokens: float | None = None class UsageTrackingCallback(ConversationCallback): @@ -21,15 +52,22 @@ class UsageTrackingCallback(ConversationCallback): Args: reporter: Reporter to write the final usage summary to. + pricing: Pricing information for cost calculation. If ``None``, + no cost data is included in the usage summary. """ - def __init__(self, reporter: Reporter = NULL_REPORTER) -> None: + def __init__( + self, + reporter: Reporter = NULL_REPORTER, + pricing: ModelPricing | None = None, + ) -> None: self._reporter = reporter - self._accumulated_usage = UsageParam() + self._pricing = pricing + self._summary = UsageSummary() @override def on_conversation_start(self, conversation: Conversation) -> None: - self._accumulated_usage = UsageParam() + self._summary = UsageSummary() @override def on_step_end( @@ -43,27 +81,29 @@ def on_step_end( @override def on_conversation_end(self, conversation: Conversation) -> None: - self._reporter.add_usage_summary(self._accumulated_usage.model_dump()) + self._reporter.add_usage_summary(self._summary) @property - def accumulated_usage(self) -> UsageParam: + def accumulated_usage(self) -> UsageSummary: """Current accumulated usage statistics.""" - return self._accumulated_usage + return self._summary def _accumulate(self, step_usage: UsageParam) -> None: - self._accumulated_usage.input_tokens = ( - self._accumulated_usage.input_tokens or 0 - ) + (step_usage.input_tokens or 0) - self._accumulated_usage.output_tokens = ( - self._accumulated_usage.output_tokens or 0 - ) + (step_usage.output_tokens or 0) - self._accumulated_usage.cache_creation_input_tokens = ( - self._accumulated_usage.cache_creation_input_tokens or 0 + # Add step tokens to running totals (None counts as 0) + self._summary.input_tokens = (self._summary.input_tokens or 0) + ( + step_usage.input_tokens or 0 + ) + self._summary.output_tokens = (self._summary.output_tokens or 0) + ( + step_usage.output_tokens or 0 + ) + self._summary.cache_creation_input_tokens = ( + self._summary.cache_creation_input_tokens or 0 ) + (step_usage.cache_creation_input_tokens or 0) - self._accumulated_usage.cache_read_input_tokens = ( - self._accumulated_usage.cache_read_input_tokens or 0 + self._summary.cache_read_input_tokens = ( + self._summary.cache_read_input_tokens or 0 ) + (step_usage.cache_read_input_tokens or 0) + # Record per-step token counts on the current OTel span current_span = trace.get_current_span() current_span.set_attributes( { @@ -75,3 +115,32 @@ def _accumulate(self, step_usage: UsageParam) -> None: "cache_read_input_tokens": (step_usage.cache_read_input_tokens or 0), } ) + + # Update costs from updated totals if pricing values are set + if not ( + self._pricing + and self._pricing.input_cost_per_million_tokens + and self._pricing.output_cost_per_million_tokens + ): + return + + input_cost = ( + self._summary.input_tokens + * self._pricing.input_cost_per_million_tokens + / 1e6 + ) + output_cost = ( + self._summary.output_tokens + * self._pricing.output_cost_per_million_tokens + / 1e6 + ) + self._summary.input_cost = input_cost + self._summary.output_cost = output_cost + self._summary.total_cost = input_cost + output_cost + self._summary.currency = self._pricing.currency + self._summary.input_cost_per_million_tokens = ( + self._pricing.input_cost_per_million_tokens + ) + self._summary.output_cost_per_million_tokens = ( + self._pricing.output_cost_per_million_tokens + ) diff --git a/src/askui/reporting.py b/src/askui/reporting.py index 4388a2cf..ec601286 100644 --- a/src/askui/reporting.py +++ b/src/askui/reporting.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import base64 import io import json @@ -9,14 +11,18 @@ from importlib.metadata import distributions from io import BytesIO from pathlib import Path -from typing import Any, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union from jinja2 import Template -from PIL import Image from typing_extensions import TypedDict, override from askui.utils.annotated_image import AnnotatedImage +if TYPE_CHECKING: + from PIL import Image + + from askui.models.shared.usage_tracking_callback import UsageSummary + def normalize_to_pil_images( image: Image.Image | list[Image.Image] | AnnotatedImage | None, @@ -80,15 +86,14 @@ def add_message( raise NotImplementedError @abstractmethod - def add_usage_summary(self, usage: dict[str, int | None]) -> None: + def add_usage_summary(self, usage: UsageSummary) -> None: """Add usage statistics summary to the report. - Called at the end of an act() execution with accumulated token usage. + Called at the end of an ``act()`` execution with accumulated token + usage and optional cost breakdown. Args: - usage (dict[str, int | None]): Accumulated usage statistics containing: - - input_tokens: Total input tokens sent to API - - output_tokens: Total output tokens generated + usage (UsageSummary): Accumulated usage statistics. """ raise NotImplementedError @@ -134,7 +139,7 @@ def add_message( pass @override - def add_usage_summary(self, usage: dict[str, int | None]) -> None: + def add_usage_summary(self, usage: UsageSummary) -> None: pass @override @@ -177,7 +182,7 @@ def add_message( reporter.add_message(role, content, image) @override - def add_usage_summary(self, usage: dict[str, int | None]) -> None: + def add_usage_summary(self, usage: UsageSummary) -> None: """Add usage summary to all reporters.""" for reporter in self._reporters: reporter.add_usage_summary(usage) @@ -215,7 +220,7 @@ def __init__(self, report_dir: str = "reports") -> None: self.report_dir = Path(report_dir) self.messages: list[dict[str, Any]] = [] self.system_info = self._collect_system_info() - self.usage_summary: dict[str, int | None] | None = None + self.usage_summary: UsageSummary | None = None self.cache_original_usage: dict[str, int | None] | None = None self._start_time: datetime | None = None @@ -264,7 +269,7 @@ def add_message( self.messages.append(message) @override - def add_usage_summary(self, usage: dict[str, int | None]) -> None: + def add_usage_summary(self, usage: UsageSummary) -> None: """Store usage summary for inclusion in the report.""" self.usage_summary = usage @@ -790,14 +795,14 @@ def generate(self) -> None: {% endif %} {% if usage_summary is not none %} - {% if usage_summary.get('input_tokens') is not none %} + {% if usage_summary.input_tokens is not none %} Input Tokens - {{ "{:,}".format(usage_summary.get('input_tokens')) }} + {{ "{:,}".format(usage_summary.input_tokens) }} {% if cache_original_usage and cache_original_usage.get('input_tokens') %} {% set original = cache_original_usage.get('input_tokens') %} - {% set current = usage_summary.get('input_tokens') %} + {% set current = usage_summary.input_tokens %} {% set saved = original - current %} {% if saved > 0 and original > 0 %} {% set savings_pct = (saved / original * 100) %} @@ -807,14 +812,14 @@ def generate(self) -> None: {% endif %} - {% if usage_summary.get('output_tokens') is not none %} + {% if usage_summary.output_tokens is not none %} Output Tokens - {{ "{:,}".format(usage_summary.get('output_tokens')) }} + {{ "{:,}".format(usage_summary.output_tokens) }} {% if cache_original_usage and cache_original_usage.get('output_tokens') %} {% set original = cache_original_usage.get('output_tokens') %} - {% set current = usage_summary.get('output_tokens') %} + {% set current = usage_summary.output_tokens %} {% set saved = original - current %} {% if saved > 0 and original > 0 %} {% set savings_pct = (saved / original * 100) %} @@ -824,6 +829,18 @@ def generate(self) -> None: {% endif %} + {% if usage_summary.total_cost is not none %} + + Estimated Cost (actual cost may differ) + + {{ "%.2f"|format(usage_summary.total_cost) }} {{ usage_summary.currency or 'USD' }} + + (Input: ${{ "%.2f"|format(usage_summary.input_cost_per_million_tokens or 0) }}/1M tokens, + Output: ${{ "%.2f"|format(usage_summary.output_cost_per_million_tokens or 0) }}/1M tokens) + + + + {% endif %} {% endif %} {% if cache_original_usage is not none %} {% if cache_original_usage.get('input_tokens') is not none %} @@ -980,7 +997,7 @@ def add_message( ) @override - def add_usage_summary(self, usage: dict[str, int | None]) -> None: + def add_usage_summary(self, usage: UsageSummary) -> None: """No-op for AllureReporter - usage is not tracked.""" @override diff --git a/src/askui/utils/model_pricing.py b/src/askui/utils/model_pricing.py new file mode 100644 index 00000000..03d2e254 --- /dev/null +++ b/src/askui/utils/model_pricing.py @@ -0,0 +1,81 @@ +"""Pricing information for model API calls.""" + +from pydantic import BaseModel + +_DEFAULT_PRICING: dict[str, "ModelPricing"] = {} + + +class ModelPricing(BaseModel): + """Cost per 1 million tokens for a model. + + Args: + input_cost_per_million_tokens (float): Cost per 1M input tokens in Currency. + output_cost_per_million_tokens (float): Cost per 1M output tokens in Currency. + currency (str): descriptor of the currency. Defaults to ``"USD"``. + """ + + input_cost_per_million_tokens: float + output_cost_per_million_tokens: float + currency: str = "USD" + + @classmethod + def for_model( + cls, + model_id: str, + input_cost_per_million_tokens: float | None = None, + output_cost_per_million_tokens: float | None = None, + ) -> "ModelPricing | None": + """Resolve pricing for a model. + + If both cost parameters are provided, creates a ``ModelPricing`` + with those values. Otherwise, looks up built-in defaults by + ``model_id``. + + Args: + model_id (str): The model identifier. + input_cost_per_million_tokens (float | None, optional): Override + cost in USD per 1M input tokens. + output_cost_per_million_tokens (float | None, optional): Override + cost in USD per 1M output tokens. + + Returns: + ModelPricing | None: Resolved pricing, or ``None`` if no match + and no overrides provided. + """ + if ( + input_cost_per_million_tokens is not None + and output_cost_per_million_tokens is not None + ): + return cls( + input_cost_per_million_tokens=input_cost_per_million_tokens, + output_cost_per_million_tokens=output_cost_per_million_tokens, + ) + return _DEFAULT_PRICING.get(model_id) + + +# taken from: https://platform.claude.com/docs/en/about-claude/models/overview +# last accessed: March 10, 2026 +_DEFAULT_PRICING.update( + { + "claude-haiku-4-5-20251001": ModelPricing( + input_cost_per_million_tokens=1.0, + output_cost_per_million_tokens=5.0, + ), + "claude-sonnet-4-5-20250929": ModelPricing( + input_cost_per_million_tokens=3.0, + output_cost_per_million_tokens=15.0, + ), + "claude-opus-4-5-20251101": ModelPricing( + input_cost_per_million_tokens=5.0, + output_cost_per_million_tokens=25.0, + ), + "claude-sonnet-4-6": ModelPricing( + input_cost_per_million_tokens=3.0, + output_cost_per_million_tokens=15.0, + ), + "claude-opus-4-6": ModelPricing( + input_cost_per_million_tokens=5.0, + output_cost_per_million_tokens=25.0, + ), + } +) diff --git a/tests/unit/model_providers/__init__.py b/tests/unit/model_providers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/model_providers/test_model_pricing.py b/tests/unit/model_providers/test_model_pricing.py new file mode 100644 index 00000000..7fa00c6d --- /dev/null +++ b/tests/unit/model_providers/test_model_pricing.py @@ -0,0 +1,142 @@ +"""Unit tests for model pricing resolution and cost calculation.""" + +from unittest.mock import MagicMock + +import pytest + +from askui.models.shared.agent_message_param import UsageParam +from askui.models.shared.usage_tracking_callback import ( + UsageSummary, + UsageTrackingCallback, +) +from askui.utils.model_pricing import ModelPricing + + +class TestModelPricingForModel: + @pytest.mark.parametrize( + ("model_id", "expected_input", "expected_output"), + [ + ("claude-haiku-4-5-20251001", 1.0, 5.0), + ("claude-sonnet-4-5-20250929", 3.0, 15.0), + ("claude-opus-4-5-20251101", 5.0, 25.0), + ("claude-sonnet-4-6", 3.0, 15.0), + ("claude-opus-4-6", 5.0, 25.0), + ], + ) + def test_known_model_returns_default_pricing( + self, + model_id: str, + expected_input: float, + expected_output: float, + ) -> None: + pricing = ModelPricing.for_model(model_id) + assert pricing is not None + assert pricing.input_cost_per_million_tokens == expected_input + assert pricing.output_cost_per_million_tokens == expected_output + + @pytest.mark.parametrize( + "model_id", + ["unknown-model-v1", "", "claude-sonnet-4"], + ) + def test_unknown_model_returns_none(self, model_id: str) -> None: + assert ModelPricing.for_model(model_id) is None + + @pytest.mark.parametrize( + ("model_id", "input_cost", "output_cost"), + [ + ("claude-sonnet-4-6", 99.0, 199.0), + ("unknown-model", 1.0, 2.0), + ], + ) + def test_override_costs( + self, + model_id: str, + input_cost: float, + output_cost: float, + ) -> None: + pricing = ModelPricing.for_model( + model_id, + input_cost_per_million_tokens=input_cost, + output_cost_per_million_tokens=output_cost, + ) + assert pricing is not None + assert pricing.input_cost_per_million_tokens == input_cost + assert pricing.output_cost_per_million_tokens == output_cost + + +def _get_usage_summary(reporter_mock: MagicMock) -> UsageSummary: + return reporter_mock.add_usage_summary.call_args[0][0] # type: ignore[no-any-return] + + +class TestUsageTrackingCallbackCost: + def _make_callback( + self, pricing: ModelPricing | None = None + ) -> tuple[UsageTrackingCallback, MagicMock]: + reporter = MagicMock() + callback = UsageTrackingCallback(reporter=reporter, pricing=pricing) + return callback, reporter + + @pytest.mark.parametrize( + ( + "input_tokens", + "output_tokens", + "input_rate", + "output_rate", + "expected_input_cost", + "expected_output_cost", + ), + [ + (1_000_000, 100_000, 3.0, 15.0, 3.0, 1.5), + (50_000, 10_000, 15.0, 75.0, 0.75, 0.75), + (0, 0, 3.0, 15.0, 0.0, 0.0), + ], + ) + def test_cost_calculation( + self, + input_tokens: int, + output_tokens: int, + input_rate: float, + output_rate: float, + expected_input_cost: float, + expected_output_cost: float, + ) -> None: + pricing = ModelPricing( + input_cost_per_million_tokens=input_rate, + output_cost_per_million_tokens=output_rate, + ) + callback, reporter = self._make_callback(pricing) + callback._accumulate( + UsageParam(input_tokens=input_tokens, output_tokens=output_tokens) + ) + callback.on_conversation_end(MagicMock()) + + summary = _get_usage_summary(reporter) + assert summary.input_cost == pytest.approx(expected_input_cost) + assert summary.output_cost == pytest.approx(expected_output_cost) + assert summary.total_cost == pytest.approx( + expected_input_cost + expected_output_cost + ) + assert summary.currency == "USD" + assert summary.input_cost_per_million_tokens == input_rate + assert summary.output_cost_per_million_tokens == output_rate + + def test_no_cost_when_pricing_none(self) -> None: + callback, reporter = self._make_callback(pricing=None) + callback._accumulate(UsageParam(input_tokens=500, output_tokens=200)) + callback.on_conversation_end(MagicMock()) + + summary = _get_usage_summary(reporter) + assert summary.total_cost is None + assert summary.currency is None + + def test_none_tokens_treated_as_zero(self) -> None: + pricing = ModelPricing( + input_cost_per_million_tokens=3.0, + output_cost_per_million_tokens=15.0, + ) + callback, reporter = self._make_callback(pricing) + callback._accumulate(UsageParam()) + callback.on_conversation_end(MagicMock()) + + summary = _get_usage_summary(reporter) + assert summary.total_cost == 0.0