From bab750c85653956ed441c09108df55f2c6fabafa Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Tue, 3 Nov 2026 06:41:19 -0500 Subject: [PATCH] Python: Count Gemini thinking tokens in Vertex AI completion usage --- .../ai/google/vertex_ai/services/utils.py | 21 ++++++++ .../services/vertex_ai_chat_completion.py | 3 +- .../services/vertex_ai_text_completion.py | 3 +- .../services/test_vertex_ai_utils.py | 48 ++++++++++++++++++- 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/google/vertex_ai/services/utils.py b/python/semantic_kernel/connectors/ai/google/vertex_ai/services/utils.py index f027880e1faf..e792e8e04290 100644 --- a/python/semantic_kernel/connectors/ai/google/vertex_ai/services/utils.py +++ b/python/semantic_kernel/connectors/ai/google/vertex_ai/services/utils.py @@ -190,3 +190,24 @@ def _create_image_part(image_content: ImageContent) -> Part: raise ServiceInvalidRequestError( "ImageContent without data_uri in User message while formatting chat history for Google AI" ) + + +def get_completion_token_count(usage_metadata: Any) -> int | None: + """Get the number of tokens the model generated, including its thinking tokens. + + Vertex AI reports the thinking tokens in ``thoughts_token_count``, apart from ``candidates_token_count``, + but bills them as output tokens, so both count toward the completion tokens. + + Args: + usage_metadata: The usage metadata of a response, if it has one. + + Returns: + The completion token count, or None if the response reports neither count. + """ + if not usage_metadata: + return None + candidates_tokens = usage_metadata.candidates_token_count + thoughts_tokens = usage_metadata.thoughts_token_count + if candidates_tokens is None and thoughts_tokens is None: + return None + return (candidates_tokens or 0) + (thoughts_tokens or 0) diff --git a/python/semantic_kernel/connectors/ai/google/vertex_ai/services/vertex_ai_chat_completion.py b/python/semantic_kernel/connectors/ai/google/vertex_ai/services/vertex_ai_chat_completion.py index 4cff1811c2af..7a183bfc3b4c 100644 --- a/python/semantic_kernel/connectors/ai/google/vertex_ai/services/vertex_ai_chat_completion.py +++ b/python/semantic_kernel/connectors/ai/google/vertex_ai/services/vertex_ai_chat_completion.py @@ -22,6 +22,7 @@ format_assistant_message, format_tool_message, format_user_message, + get_completion_token_count, update_settings_from_function_choice_configuration, ) from semantic_kernel.connectors.ai.google.vertex_ai.services.vertex_ai_base import VertexAIBase @@ -355,7 +356,7 @@ def _get_metadata_from_response(self, response: GenerationResponse) -> dict[str, "prompt_feedback": response.prompt_feedback, "usage": CompletionUsage( prompt_tokens=response.usage_metadata.prompt_token_count, - completion_tokens=response.usage_metadata.candidates_token_count, + completion_tokens=get_completion_token_count(response.usage_metadata), ), } diff --git a/python/semantic_kernel/connectors/ai/google/vertex_ai/services/vertex_ai_text_completion.py b/python/semantic_kernel/connectors/ai/google/vertex_ai/services/vertex_ai_text_completion.py index aa47ebc055f9..6a0f622ff4cc 100644 --- a/python/semantic_kernel/connectors/ai/google/vertex_ai/services/vertex_ai_text_completion.py +++ b/python/semantic_kernel/connectors/ai/google/vertex_ai/services/vertex_ai_text_completion.py @@ -11,6 +11,7 @@ from vertexai.generative_models import Candidate, GenerationResponse, GenerativeModel from semantic_kernel.connectors.ai.completion_usage import CompletionUsage +from semantic_kernel.connectors.ai.google.vertex_ai.services.utils import get_completion_token_count from semantic_kernel.connectors.ai.google.vertex_ai.services.vertex_ai_base import VertexAIBase from semantic_kernel.connectors.ai.google.vertex_ai.vertex_ai_prompt_execution_settings import ( VertexAITextPromptExecutionSettings, @@ -191,7 +192,7 @@ def _get_metadata_from_response(self, response: GenerationResponse) -> dict[str, "prompt_feedback": response.prompt_feedback, "usage": CompletionUsage( prompt_tokens=response.usage_metadata.prompt_token_count, - completion_tokens=response.usage_metadata.candidates_token_count, + completion_tokens=get_completion_token_count(response.usage_metadata), ), } diff --git a/python/tests/unit/connectors/ai/google/vertex_ai/services/test_vertex_ai_utils.py b/python/tests/unit/connectors/ai/google/vertex_ai/services/test_vertex_ai_utils.py index f05ec0d11187..f2d568475b54 100644 --- a/python/tests/unit/connectors/ai/google/vertex_ai/services/test_vertex_ai_utils.py +++ b/python/tests/unit/connectors/ai/google/vertex_ai/services/test_vertex_ai_utils.py @@ -2,12 +2,14 @@ import pytest from google.cloud.aiplatform_v1beta1.types.content import Candidate -from vertexai.generative_models import FunctionDeclaration, Part +from google.cloud.aiplatform_v1beta1.types.prediction_service import GenerateContentResponse +from vertexai.generative_models import FunctionDeclaration, GenerationResponse, Part from semantic_kernel.connectors.ai.google.vertex_ai.services.utils import ( finish_reason_from_vertex_ai_to_semantic_kernel, format_assistant_message, format_user_message, + get_completion_token_count, kernel_function_metadata_to_vertex_ai_function_call_format, ) from semantic_kernel.contents.chat_message_content import ChatMessageContent @@ -221,3 +223,47 @@ def test_vertex_ai_function_call_format_skips_params_excluded_from_function_choi schema = type(result).to_dict(result)["parameters"] assert list(schema["properties"]) == ["query"] assert schema["required"] == ["query"] + + +def _usage_response(candidates_tokens, thoughts_tokens) -> GenerationResponse: + return GenerationResponse._from_gapic( + GenerateContentResponse( + usage_metadata=GenerateContentResponse.UsageMetadata( + prompt_token_count=5, + candidates_token_count=candidates_tokens, + thoughts_token_count=thoughts_tokens, + total_token_count=55, + ) + ) + ) + + +@pytest.mark.parametrize( + ("candidates_tokens", "thoughts_tokens", "expected"), + [(10, 40, 50), (10, 0, 10), (0, 40, 40), (0, 0, 0)], +) +def test_get_completion_token_count_includes_thinking_tokens(candidates_tokens, thoughts_tokens, expected) -> None: + usage_metadata = _usage_response(candidates_tokens, thoughts_tokens).usage_metadata + assert get_completion_token_count(usage_metadata) == expected + + +def test_get_completion_token_count_without_usage_metadata() -> None: + assert get_completion_token_count(None) is None + + +def test_chat_completion_usage_counts_thinking_tokens(vertex_ai_unit_test_env) -> None: + from semantic_kernel.connectors.ai.google.vertex_ai.services.vertex_ai_chat_completion import ( + VertexAIChatCompletion, + ) + + usage = VertexAIChatCompletion()._get_metadata_from_response(_usage_response(10, 40))["usage"] + assert (usage.prompt_tokens, usage.completion_tokens) == (5, 50) + + +def test_text_completion_usage_counts_thinking_tokens(vertex_ai_unit_test_env) -> None: + from semantic_kernel.connectors.ai.google.vertex_ai.services.vertex_ai_text_completion import ( + VertexAITextCompletion, + ) + + usage = VertexAITextCompletion()._get_metadata_from_response(_usage_response(10, 40))["usage"] + assert (usage.prompt_tokens, usage.completion_tokens) == (5, 50)