diff --git a/python/semantic_kernel/connectors/ai/chat_completion_client_base.py b/python/semantic_kernel/connectors/ai/chat_completion_client_base.py index 974d59af92be..9b595350c11f 100644 --- a/python/semantic_kernel/connectors/ai/chat_completion_client_base.py +++ b/python/semantic_kernel/connectors/ai/chat_completion_client_base.py @@ -16,6 +16,7 @@ from semantic_kernel.contents.annotation_content import AnnotationContent from semantic_kernel.contents.file_reference_content import FileReferenceContent from semantic_kernel.contents.function_call_content import FunctionCallContent +from semantic_kernel.contents.utils.finish_reason import FinishReason from semantic_kernel.exceptions.service_exceptions import ServiceInvalidExecutionSettingsError from semantic_kernel.services.ai_service_client_base import AIServiceClientBase from semantic_kernel.utils.telemetry.model_diagnostics.gen_ai_attributes import AVAILABLE_FUNCTIONS @@ -139,7 +140,7 @@ async def get_chat_message_contents( completions = await self._inner_get_chat_message_contents(chat_history, settings) # Get the function call contents from the chat message. There is only one chat message, # which should be checked in the `_verify_function_choice_settings` method. - function_calls = [item for item in completions[0].items if isinstance(item, FunctionCallContent)] + function_calls = self._drop_unwritten_function_calls(completions[0]) if (fc_count := len(function_calls)) == 0: return completions @@ -173,6 +174,25 @@ async def get_chat_message_contents( self._reset_function_choice_settings(settings) return await self._inner_get_chat_message_contents(chat_history, settings) + @staticmethod + def _drop_unwritten_function_calls( + message: "ChatMessageContent | StreamingChatMessageContent", + ) -> list[FunctionCallContent]: + """Return the function calls of a message that should be invoked. + + When the model stopped because it hit the token limit, a function call whose arguments + were never written would otherwise run with its defaults, so it is removed from the message. + Calls with any arguments, and all calls of a reply that was not cut off, are kept as before. + """ + function_calls = [item for item in message.items if isinstance(item, FunctionCallContent)] + if message.finish_reason != FinishReason.LENGTH: + return function_calls + unwritten = [call for call in function_calls if not call.arguments] + if unwritten: + logger.warning("Skipping tool calls with no arguments because the reply was cut off by the length limit.") + message.items = [item for item in message.items if not any(item is call for call in unwritten)] + return [call for call in function_calls if call.arguments] + async def get_chat_message_content( self, chat_history: "ChatHistory", settings: "PromptExecutionSettings", **kwargs: Any ) -> "ChatMessageContent | None": @@ -277,7 +297,9 @@ async def get_streaming_chat_message_contents( # the full completion depending on the prompt, the message may contain both function call # content and others full_completion: StreamingChatMessageContent = reduce(lambda x, y: x + y, all_messages) - function_calls = [item for item in full_completion.items if isinstance(item, FunctionCallContent)] + function_calls = self._drop_unwritten_function_calls(full_completion) + if not function_calls: + return chat_history.add_message(message=full_completion) fc_count = len(function_calls) diff --git a/python/tests/unit/connectors/ai/open_ai/services/test_openai_chat_completion_truncated_tool_call.py b/python/tests/unit/connectors/ai/open_ai/services/test_openai_chat_completion_truncated_tool_call.py new file mode 100644 index 000000000000..4a7112b6156c --- /dev/null +++ b/python/tests/unit/connectors/ai/open_ai/services/test_openai_chat_completion_truncated_tool_call.py @@ -0,0 +1,130 @@ +# Copyright (c) Microsoft. All rights reserved. + +import json + +import httpx +from openai import AsyncOpenAI + +from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior +from semantic_kernel.connectors.ai.open_ai import OpenAIChatCompletion +from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.contents.chat_history import ChatHistory +from semantic_kernel.functions.kernel_function_decorator import kernel_function +from semantic_kernel.kernel import Kernel + + +class _Plugin: + def __init__(self) -> None: + self.calls: list[str] = [] + + @kernel_function(name="set_mode") + def set_mode(self, mode: str = "default") -> str: + self.calls.append(mode) + return "done" + + +def _service(handler) -> OpenAIChatCompletion: + client = AsyncOpenAI( + api_key="sk-" + "a" * 48, + base_url="http://mock/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), + ) + return OpenAIChatCompletion(ai_model_id="gpt-4o-mini", async_client=client) + + +def _stream_handler(arguments: str, finish_reason: str): + body = ( + 'data: {"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1",' + '"type":"function","function":{"name":"set_mode","arguments":' + json.dumps(arguments) + "}}]}}]}\n\n" + 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"' + finish_reason + '"}]}\n\n' + "data: [DONE]\n\n" + ) + + def handler(request: httpx.Request) -> httpx.Response: + if json.loads(request.content).get("messages", [])[-1]["role"] == "tool": + done = 'data: {"choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n' + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=done.encode()) + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=body.encode()) + + return handler + + +def _json_handler(arguments: str, finish_reason: str): + def handler(request: httpx.Request) -> httpx.Response: + if json.loads(request.content).get("messages", [])[-1]["role"] == "tool": + message = {"role": "assistant", "content": "ok"} + choice = {"index": 0, "message": message, "finish_reason": "stop"} + else: + message = { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "set_mode", "arguments": arguments}} + ], + } + choice = {"index": 0, "message": message, "finish_reason": finish_reason} + payload = { + "id": "x", + "object": "chat.completion", + "created": 0, + "model": "gpt-4o-mini", + "choices": [choice], + } + return httpx.Response(200, json=payload) + + return handler + + +async def _run_streaming(service, plugin) -> None: + kernel = Kernel() + kernel.add_plugin(plugin, plugin_name="p") + settings = PromptExecutionSettings(function_choice_behavior=FunctionChoiceBehavior.Auto()) + history = ChatHistory() + history.add_user_message("do it") + async for _ in service.get_streaming_chat_message_contents(history, settings, kernel=kernel): + pass + + +async def _run_non_streaming(service, plugin) -> None: + kernel = Kernel() + kernel.add_plugin(plugin, plugin_name="p") + settings = PromptExecutionSettings(function_choice_behavior=FunctionChoiceBehavior.Auto()) + history = ChatHistory() + history.add_user_message("do it") + await service.get_chat_message_contents(history, settings, kernel=kernel) + + +async def test_streaming_skips_tool_call_with_no_arguments_when_cut_off(): + plugin = _Plugin() + await _run_streaming(_service(_stream_handler("", "length")), plugin) + assert plugin.calls == [] + + +async def test_non_streaming_skips_tool_call_with_no_arguments_when_cut_off(): + plugin = _Plugin() + await _run_non_streaming(_service(_json_handler("", "length")), plugin) + assert plugin.calls == [] + + +async def test_streaming_runs_tool_call_with_arguments_when_cut_off_afterwards(): + plugin = _Plugin() + await _run_streaming(_service(_stream_handler('{"mode": "silent"}', "length")), plugin) + assert plugin.calls == ["silent"] + + +async def test_non_streaming_runs_tool_call_with_arguments_when_cut_off_afterwards(): + plugin = _Plugin() + await _run_non_streaming(_service(_json_handler('{"mode": "silent"}', "length")), plugin) + assert plugin.calls == ["silent"] + + +async def test_streaming_runs_tool_call_with_no_arguments_when_not_cut_off(): + plugin = _Plugin() + await _run_streaming(_service(_stream_handler("", "tool_calls")), plugin) + assert plugin.calls == ["default"] + + +async def test_non_streaming_runs_tool_call_with_no_arguments_when_not_cut_off(): + plugin = _Plugin() + await _run_non_streaming(_service(_json_handler("", "tool_calls")), plugin) + assert plugin.calls == ["default"]