diff --git a/python/semantic_kernel/connectors/ai/onnx/utils.py b/python/semantic_kernel/connectors/ai/onnx/utils.py index e9bbe40ee074..cd2f8521647f 100644 --- a/python/semantic_kernel/connectors/ai/onnx/utils.py +++ b/python/semantic_kernel/connectors/ai/onnx/utils.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft. All rights reserved. from enum import Enum -from semantic_kernel.contents import AuthorRole, ChatHistory, ImageContent, TextContent +from semantic_kernel.contents import AuthorRole, ChatHistory, ChatMessageContent, ImageContent, TextContent from semantic_kernel.contents.audio_content import AudioContent from semantic_kernel.exceptions import ServiceException, ServiceInvalidRequestError @@ -104,12 +104,8 @@ def phi3v_template(history: ChatHistory) -> str: if message.role == AuthorRole.SYSTEM: phi3v_input += f"<|system|>\n{message.content}<|end|>\n" if message.role == AuthorRole.USER: - for item in message.items: - if isinstance(item, TextContent): - phi3v_input += f"<|user|>\n{item.text}<|end|>\n" - if isinstance(item, ImageContent): - phi3v_input += f"<|image_{image_count + 1}|>\n" - image_count += 1 + user_content, image_count, _ = _multimodal_user_content(message, image_count) + phi3v_input += f"<|user|>\n{user_content}<|end|>\n" if message.role == AuthorRole.ASSISTANT: phi3v_input += f"<|assistant|>\n{message.content}<|end|>\n" phi3v_input += "<|assistant|>\n" @@ -133,22 +129,53 @@ def phi4mm_template(history: ChatHistory) -> str: if message.role == AuthorRole.SYSTEM: phi4mm_input += f"<|system|>\n{message.content}<|end|>\n" if message.role == AuthorRole.USER: - for item in message.items: - if isinstance(item, TextContent): - phi4mm_input += f"<|user|>\n{item.text}<|end|>\n" - # Currently only one image is supported in Onnx - if isinstance(item, ImageContent): - phi4mm_input += f"<|image_{image_count + 1}|>\n" - image_count += 1 - if isinstance(item, AudioContent): - phi4mm_input += f"<|audio_{audio_count + 1}|>\n" - audio_count += 1 + user_content, image_count, audio_count = _multimodal_user_content( + message, image_count, audio_count, include_audio=True + ) + phi4mm_input += f"<|user|>\n{user_content}<|end|>\n" if message.role == AuthorRole.ASSISTANT: phi4mm_input += f"<|assistant|>\n{message.content}<|end|>\n" phi4mm_input += "<|assistant|>\n" return phi4mm_input +def _multimodal_user_content( + message: ChatMessageContent, + image_count: int, + audio_count: int = 0, + include_audio: bool = False, +) -> tuple[str, int, int]: + """Builds the content of a single user turn for the Phi-3-vision and Phi-4-multimodal templates. + + All content of the message goes into one user turn: the image placeholders first, then the audio + placeholders, each on its own line, then the text. This is the prompt format given in the model + cards and used by the onnxruntime-genai examples. Placeholders are numbered across the whole chat + history, starting at 1. + + Args: + message (ChatMessageContent): The user message. + image_count (int): The number of images in the chat history before this message. + audio_count (int): The number of audio clips in the chat history before this message. + include_audio (bool): Whether to emit audio placeholders. + + Returns: + tuple[str, int, int]: The user content and the updated image and audio counts. + """ + image_tags = "" + audio_tags = "" + texts: list[str] = [] + for item in message.items: + if isinstance(item, TextContent): + texts.append(item.text) + elif isinstance(item, ImageContent): + image_count += 1 + image_tags += f"<|image_{image_count}|>\n" + elif include_audio and isinstance(item, AudioContent): + audio_count += 1 + audio_tags += f"<|audio_{audio_count}|>\n" + return image_tags + audio_tags + "\n".join(texts), image_count, audio_count + + def gemma_template(history: ChatHistory) -> str: """Generates a formatted string for the Gemma model based on the provided chat history. diff --git a/python/tests/unit/connectors/ai/onnx/services/test_onnx_utils.py b/python/tests/unit/connectors/ai/onnx/services/test_onnx_utils.py index 04ccd085c538..5332a323b448 100644 --- a/python/tests/unit/connectors/ai/onnx/services/test_onnx_utils.py +++ b/python/tests/unit/connectors/ai/onnx/services/test_onnx_utils.py @@ -10,6 +10,9 @@ ) from semantic_kernel.contents import AudioContent, AuthorRole, ChatHistory, ImageContent, TextContent +IMAGE_URL = "http://example.com/image.png" +AUDIO_URL = "http://example.com/audio.mp3" + def test_phi3v_template_with_text_and_image(): history = ChatHistory( @@ -17,7 +20,7 @@ def test_phi3v_template_with_text_and_image(): {"role": AuthorRole.SYSTEM, "content": "System message"}, { "role": AuthorRole.USER, - "items": [TextContent(text="User text message"), ImageContent(url="http://example.com/image.png")], + "items": [TextContent(text="User text message"), ImageContent(url=IMAGE_URL)], }, {"role": AuthorRole.ASSISTANT, "content": "Assistant message"}, ] @@ -25,8 +28,7 @@ def test_phi3v_template_with_text_and_image(): expected_output = ( "<|system|>\nSystem message<|end|>\n" - "<|user|>\nUser text message<|end|>\n" - "<|image_1|>\n" + "<|user|>\n<|image_1|>\nUser text message<|end|>\n" "<|assistant|>\nAssistant message<|end|>\n" "<|assistant|>\n" ) @@ -34,6 +36,78 @@ def test_phi3v_template_with_text_and_image(): assert phi3v_template(history) == expected_output +def test_phi3v_template_with_image_and_text(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [ImageContent(url=IMAGE_URL), TextContent(text="Describe this image.")], + }, + ] + ) + + expected_output = "<|user|>\n<|image_1|>\nDescribe this image.<|end|>\n<|assistant|>\n" + + assert phi3v_template(history) == expected_output + + +def test_phi3v_template_with_multiple_images(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [ + ImageContent(url=IMAGE_URL), + ImageContent(url=IMAGE_URL), + TextContent(text="Compare these images."), + ], + }, + ] + ) + + expected_output = "<|user|>\n<|image_1|>\n<|image_2|>\nCompare these images.<|end|>\n<|assistant|>\n" + + assert phi3v_template(history) == expected_output + + +def test_phi3v_template_numbers_images_across_turns(): + history = ChatHistory( + messages=[ + {"role": AuthorRole.USER, "items": [ImageContent(url=IMAGE_URL), TextContent(text="First")]}, + {"role": AuthorRole.ASSISTANT, "content": "Answer"}, + {"role": AuthorRole.USER, "items": [TextContent(text="Second"), ImageContent(url=IMAGE_URL)]}, + {"role": AuthorRole.ASSISTANT, "content": "Answer"}, + {"role": AuthorRole.USER, "items": [TextContent(text="Third")]}, + ] + ) + + expected_output = ( + "<|user|>\n<|image_1|>\nFirst<|end|>\n" + "<|assistant|>\nAnswer<|end|>\n" + "<|user|>\n<|image_2|>\nSecond<|end|>\n" + "<|assistant|>\nAnswer<|end|>\n" + "<|user|>\nThird<|end|>\n" + "<|assistant|>\n" + ) + + assert phi3v_template(history) == expected_output + + +def test_phi3v_template_merges_text_items_into_one_user_turn(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [TextContent(text="Line one"), ImageContent(url=IMAGE_URL), TextContent(text="Line two")], + }, + ] + ) + + expected_output = "<|user|>\n<|image_1|>\nLine one\nLine two<|end|>\n<|assistant|>\n" + + assert phi3v_template(history) == expected_output + + def test_phi4mm_template_with_text_and_image(): history = ChatHistory( messages=[ @@ -42,8 +116,8 @@ def test_phi4mm_template_with_text_and_image(): "role": AuthorRole.USER, "items": [ TextContent(text="User text message"), - ImageContent(url="http://example.com/image.png"), - AudioContent(url="http://example.com/audio.mp3"), + ImageContent(url=IMAGE_URL), + AudioContent(url=AUDIO_URL), ], }, {"role": AuthorRole.ASSISTANT, "content": "Assistant message"}, @@ -52,9 +126,7 @@ def test_phi4mm_template_with_text_and_image(): expected_output = ( "<|system|>\nSystem message<|end|>\n" - "<|user|>\nUser text message<|end|>\n" - "<|image_1|>\n" - "<|audio_1|>\n" + "<|user|>\n<|image_1|>\n<|audio_1|>\nUser text message<|end|>\n" "<|assistant|>\nAssistant message<|end|>\n" "<|assistant|>\n" ) @@ -62,6 +134,85 @@ def test_phi4mm_template_with_text_and_image(): assert phi4mm_template(history) == expected_output +def test_phi4mm_template_with_image_and_text(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [ImageContent(url=IMAGE_URL), TextContent(text="Describe this image.")], + }, + ] + ) + + expected_output = "<|user|>\n<|image_1|>\nDescribe this image.<|end|>\n<|assistant|>\n" + + assert phi4mm_template(history) == expected_output + + +def test_phi4mm_template_with_multiple_images(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [ + TextContent(text="Summarize the content of the images."), + ImageContent(url=IMAGE_URL), + ImageContent(url=IMAGE_URL), + ImageContent(url=IMAGE_URL), + ], + }, + ] + ) + + expected_output = ( + "<|user|>\n<|image_1|>\n<|image_2|>\n<|image_3|>\nSummarize the content of the images.<|end|>\n<|assistant|>\n" + ) + + assert phi4mm_template(history) == expected_output + + +def test_phi4mm_template_with_text_and_audio(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [TextContent(text="Transcribe the audio clip into text."), AudioContent(url=AUDIO_URL)], + }, + ] + ) + + expected_output = "<|user|>\n<|audio_1|>\nTranscribe the audio clip into text.<|end|>\n<|assistant|>\n" + + assert phi4mm_template(history) == expected_output + + +def test_phi4mm_template_with_audio_only(): + history = ChatHistory(messages=[{"role": AuthorRole.USER, "items": [AudioContent(url=AUDIO_URL)]}]) + + expected_output = "<|user|>\n<|audio_1|>\n<|end|>\n<|assistant|>\n" + + assert phi4mm_template(history) == expected_output + + +def test_phi4mm_template_puts_images_before_audio(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [ + AudioContent(url=AUDIO_URL), + ImageContent(url=IMAGE_URL), + ImageContent(url=IMAGE_URL), + ], + }, + ] + ) + + expected_output = "<|user|>\n<|image_1|>\n<|image_2|>\n<|audio_1|>\n<|end|>\n<|assistant|>\n" + + assert phi4mm_template(history) == expected_output + + def test_phi3_template_with_only_text(): history = ChatHistory(messages=[{"role": AuthorRole.USER, "items": [TextContent(text="User text message")]}])