Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -190,3 +190,24 @@ def _create_image_part(image_content: ImageContent) -> Part:
raise ServiceInvalidRequestError(
"ImageContent without data_uri in User message while formatting chat history for Google AI"
)


def get_completion_token_count(usage_metadata: Any) -> int | None:
"""Get the number of tokens the model generated, including its thinking tokens.

Vertex AI reports the thinking tokens in ``thoughts_token_count``, apart from ``candidates_token_count``,
but bills them as output tokens, so both count toward the completion tokens.

Args:
usage_metadata: The usage metadata of a response, if it has one.

Returns:
The completion token count, or None if the response reports neither count.
"""
if not usage_metadata:
return None
candidates_tokens = usage_metadata.candidates_token_count
thoughts_tokens = usage_metadata.thoughts_token_count
if candidates_tokens is None and thoughts_tokens is None:
return None
return (candidates_tokens or 0) + (thoughts_tokens or 0)
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
format_assistant_message,
format_tool_message,
format_user_message,
get_completion_token_count,
update_settings_from_function_choice_configuration,
)
from semantic_kernel.connectors.ai.google.vertex_ai.services.vertex_ai_base import VertexAIBase
Expand Down Expand Up @@ -355,7 +356,7 @@ def _get_metadata_from_response(self, response: GenerationResponse) -> dict[str,
"prompt_feedback": response.prompt_feedback,
"usage": CompletionUsage(
prompt_tokens=response.usage_metadata.prompt_token_count,
completion_tokens=response.usage_metadata.candidates_token_count,
completion_tokens=get_completion_token_count(response.usage_metadata),
),
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from vertexai.generative_models import Candidate, GenerationResponse, GenerativeModel

from semantic_kernel.connectors.ai.completion_usage import CompletionUsage
from semantic_kernel.connectors.ai.google.vertex_ai.services.utils import get_completion_token_count
from semantic_kernel.connectors.ai.google.vertex_ai.services.vertex_ai_base import VertexAIBase
from semantic_kernel.connectors.ai.google.vertex_ai.vertex_ai_prompt_execution_settings import (
VertexAITextPromptExecutionSettings,
Expand Down Expand Up @@ -191,7 +192,7 @@ def _get_metadata_from_response(self, response: GenerationResponse) -> dict[str,
"prompt_feedback": response.prompt_feedback,
"usage": CompletionUsage(
prompt_tokens=response.usage_metadata.prompt_token_count,
completion_tokens=response.usage_metadata.candidates_token_count,
completion_tokens=get_completion_token_count(response.usage_metadata),
),
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@

import pytest
from google.cloud.aiplatform_v1beta1.types.content import Candidate
from vertexai.generative_models import FunctionDeclaration, Part
from google.cloud.aiplatform_v1beta1.types.prediction_service import GenerateContentResponse
from vertexai.generative_models import FunctionDeclaration, GenerationResponse, Part

from semantic_kernel.connectors.ai.google.vertex_ai.services.utils import (
finish_reason_from_vertex_ai_to_semantic_kernel,
format_assistant_message,
format_user_message,
get_completion_token_count,
kernel_function_metadata_to_vertex_ai_function_call_format,
)
from semantic_kernel.contents.chat_message_content import ChatMessageContent
Expand Down Expand Up @@ -221,3 +223,47 @@ def test_vertex_ai_function_call_format_skips_params_excluded_from_function_choi
schema = type(result).to_dict(result)["parameters"]
assert list(schema["properties"]) == ["query"]
assert schema["required"] == ["query"]


def _usage_response(candidates_tokens, thoughts_tokens) -> GenerationResponse:
return GenerationResponse._from_gapic(
GenerateContentResponse(
usage_metadata=GenerateContentResponse.UsageMetadata(
prompt_token_count=5,
candidates_token_count=candidates_tokens,
thoughts_token_count=thoughts_tokens,
total_token_count=55,
)
)
)


@pytest.mark.parametrize(
("candidates_tokens", "thoughts_tokens", "expected"),
[(10, 40, 50), (10, 0, 10), (0, 40, 40), (0, 0, 0)],
)
def test_get_completion_token_count_includes_thinking_tokens(candidates_tokens, thoughts_tokens, expected) -> None:
usage_metadata = _usage_response(candidates_tokens, thoughts_tokens).usage_metadata
assert get_completion_token_count(usage_metadata) == expected


def test_get_completion_token_count_without_usage_metadata() -> None:
assert get_completion_token_count(None) is None


def test_chat_completion_usage_counts_thinking_tokens(vertex_ai_unit_test_env) -> None:
from semantic_kernel.connectors.ai.google.vertex_ai.services.vertex_ai_chat_completion import (
VertexAIChatCompletion,
)

usage = VertexAIChatCompletion()._get_metadata_from_response(_usage_response(10, 40))["usage"]
assert (usage.prompt_tokens, usage.completion_tokens) == (5, 50)


def test_text_completion_usage_counts_thinking_tokens(vertex_ai_unit_test_env) -> None:
from semantic_kernel.connectors.ai.google.vertex_ai.services.vertex_ai_text_completion import (
VertexAITextCompletion,
)

usage = VertexAITextCompletion()._get_metadata_from_response(_usage_response(10, 40))["usage"]
assert (usage.prompt_tokens, usage.completion_tokens) == (5, 50)
Loading