From 30b40fda01d5c71434e4db06acc351a93164aec2 Mon Sep 17 00:00:00 2001 From: Qiming Bao Date: Sat, 7 Nov 2026 00:36:32 +0000 Subject: [PATCH] Python: Put media placeholders inside the user turn in the ONNX Phi-3-vision and Phi-4-multimodal templates phi3v_template and phi4mm_template emitted <|image_N|> / <|audio_N|> outside the <|user|> ... <|end|> turn and opened a separate user turn for every TextContent item. Build one user turn per message instead, with the image placeholders first, then the audio placeholders, then the text, which is the format in the Phi-3-vision, Phi-3.5-vision and Phi-4-multimodal model cards and in the onnxruntime-genai multimodal example. --- .../connectors/ai/onnx/utils.py | 61 +++++-- .../ai/onnx/services/test_onnx_utils.py | 167 +++++++++++++++++- 2 files changed, 203 insertions(+), 25 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/onnx/utils.py b/python/semantic_kernel/connectors/ai/onnx/utils.py index e9bbe40ee074..cd2f8521647f 100644 --- a/python/semantic_kernel/connectors/ai/onnx/utils.py +++ b/python/semantic_kernel/connectors/ai/onnx/utils.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft. All rights reserved. from enum import Enum -from semantic_kernel.contents import AuthorRole, ChatHistory, ImageContent, TextContent +from semantic_kernel.contents import AuthorRole, ChatHistory, ChatMessageContent, ImageContent, TextContent from semantic_kernel.contents.audio_content import AudioContent from semantic_kernel.exceptions import ServiceException, ServiceInvalidRequestError @@ -104,12 +104,8 @@ def phi3v_template(history: ChatHistory) -> str: if message.role == AuthorRole.SYSTEM: phi3v_input += f"<|system|>\n{message.content}<|end|>\n" if message.role == AuthorRole.USER: - for item in message.items: - if isinstance(item, TextContent): - phi3v_input += f"<|user|>\n{item.text}<|end|>\n" - if isinstance(item, ImageContent): - phi3v_input += f"<|image_{image_count + 1}|>\n" - image_count += 1 + user_content, image_count, _ = _multimodal_user_content(message, image_count) + phi3v_input += f"<|user|>\n{user_content}<|end|>\n" if message.role == AuthorRole.ASSISTANT: phi3v_input += f"<|assistant|>\n{message.content}<|end|>\n" phi3v_input += "<|assistant|>\n" @@ -133,22 +129,53 @@ def phi4mm_template(history: ChatHistory) -> str: if message.role == AuthorRole.SYSTEM: phi4mm_input += f"<|system|>\n{message.content}<|end|>\n" if message.role == AuthorRole.USER: - for item in message.items: - if isinstance(item, TextContent): - phi4mm_input += f"<|user|>\n{item.text}<|end|>\n" - # Currently only one image is supported in Onnx - if isinstance(item, ImageContent): - phi4mm_input += f"<|image_{image_count + 1}|>\n" - image_count += 1 - if isinstance(item, AudioContent): - phi4mm_input += f"<|audio_{audio_count + 1}|>\n" - audio_count += 1 + user_content, image_count, audio_count = _multimodal_user_content( + message, image_count, audio_count, include_audio=True + ) + phi4mm_input += f"<|user|>\n{user_content}<|end|>\n" if message.role == AuthorRole.ASSISTANT: phi4mm_input += f"<|assistant|>\n{message.content}<|end|>\n" phi4mm_input += "<|assistant|>\n" return phi4mm_input +def _multimodal_user_content( + message: ChatMessageContent, + image_count: int, + audio_count: int = 0, + include_audio: bool = False, +) -> tuple[str, int, int]: + """Builds the content of a single user turn for the Phi-3-vision and Phi-4-multimodal templates. + + All content of the message goes into one user turn: the image placeholders first, then the audio + placeholders, each on its own line, then the text. This is the prompt format given in the model + cards and used by the onnxruntime-genai examples. Placeholders are numbered across the whole chat + history, starting at 1. + + Args: + message (ChatMessageContent): The user message. + image_count (int): The number of images in the chat history before this message. + audio_count (int): The number of audio clips in the chat history before this message. + include_audio (bool): Whether to emit audio placeholders. + + Returns: + tuple[str, int, int]: The user content and the updated image and audio counts. + """ + image_tags = "" + audio_tags = "" + texts: list[str] = [] + for item in message.items: + if isinstance(item, TextContent): + texts.append(item.text) + elif isinstance(item, ImageContent): + image_count += 1 + image_tags += f"<|image_{image_count}|>\n" + elif include_audio and isinstance(item, AudioContent): + audio_count += 1 + audio_tags += f"<|audio_{audio_count}|>\n" + return image_tags + audio_tags + "\n".join(texts), image_count, audio_count + + def gemma_template(history: ChatHistory) -> str: """Generates a formatted string for the Gemma model based on the provided chat history. diff --git a/python/tests/unit/connectors/ai/onnx/services/test_onnx_utils.py b/python/tests/unit/connectors/ai/onnx/services/test_onnx_utils.py index 04ccd085c538..5332a323b448 100644 --- a/python/tests/unit/connectors/ai/onnx/services/test_onnx_utils.py +++ b/python/tests/unit/connectors/ai/onnx/services/test_onnx_utils.py @@ -10,6 +10,9 @@ ) from semantic_kernel.contents import AudioContent, AuthorRole, ChatHistory, ImageContent, TextContent +IMAGE_URL = "http://example.com/image.png" +AUDIO_URL = "http://example.com/audio.mp3" + def test_phi3v_template_with_text_and_image(): history = ChatHistory( @@ -17,7 +20,7 @@ def test_phi3v_template_with_text_and_image(): {"role": AuthorRole.SYSTEM, "content": "System message"}, { "role": AuthorRole.USER, - "items": [TextContent(text="User text message"), ImageContent(url="http://example.com/image.png")], + "items": [TextContent(text="User text message"), ImageContent(url=IMAGE_URL)], }, {"role": AuthorRole.ASSISTANT, "content": "Assistant message"}, ] @@ -25,8 +28,7 @@ def test_phi3v_template_with_text_and_image(): expected_output = ( "<|system|>\nSystem message<|end|>\n" - "<|user|>\nUser text message<|end|>\n" - "<|image_1|>\n" + "<|user|>\n<|image_1|>\nUser text message<|end|>\n" "<|assistant|>\nAssistant message<|end|>\n" "<|assistant|>\n" ) @@ -34,6 +36,78 @@ def test_phi3v_template_with_text_and_image(): assert phi3v_template(history) == expected_output +def test_phi3v_template_with_image_and_text(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [ImageContent(url=IMAGE_URL), TextContent(text="Describe this image.")], + }, + ] + ) + + expected_output = "<|user|>\n<|image_1|>\nDescribe this image.<|end|>\n<|assistant|>\n" + + assert phi3v_template(history) == expected_output + + +def test_phi3v_template_with_multiple_images(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [ + ImageContent(url=IMAGE_URL), + ImageContent(url=IMAGE_URL), + TextContent(text="Compare these images."), + ], + }, + ] + ) + + expected_output = "<|user|>\n<|image_1|>\n<|image_2|>\nCompare these images.<|end|>\n<|assistant|>\n" + + assert phi3v_template(history) == expected_output + + +def test_phi3v_template_numbers_images_across_turns(): + history = ChatHistory( + messages=[ + {"role": AuthorRole.USER, "items": [ImageContent(url=IMAGE_URL), TextContent(text="First")]}, + {"role": AuthorRole.ASSISTANT, "content": "Answer"}, + {"role": AuthorRole.USER, "items": [TextContent(text="Second"), ImageContent(url=IMAGE_URL)]}, + {"role": AuthorRole.ASSISTANT, "content": "Answer"}, + {"role": AuthorRole.USER, "items": [TextContent(text="Third")]}, + ] + ) + + expected_output = ( + "<|user|>\n<|image_1|>\nFirst<|end|>\n" + "<|assistant|>\nAnswer<|end|>\n" + "<|user|>\n<|image_2|>\nSecond<|end|>\n" + "<|assistant|>\nAnswer<|end|>\n" + "<|user|>\nThird<|end|>\n" + "<|assistant|>\n" + ) + + assert phi3v_template(history) == expected_output + + +def test_phi3v_template_merges_text_items_into_one_user_turn(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [TextContent(text="Line one"), ImageContent(url=IMAGE_URL), TextContent(text="Line two")], + }, + ] + ) + + expected_output = "<|user|>\n<|image_1|>\nLine one\nLine two<|end|>\n<|assistant|>\n" + + assert phi3v_template(history) == expected_output + + def test_phi4mm_template_with_text_and_image(): history = ChatHistory( messages=[ @@ -42,8 +116,8 @@ def test_phi4mm_template_with_text_and_image(): "role": AuthorRole.USER, "items": [ TextContent(text="User text message"), - ImageContent(url="http://example.com/image.png"), - AudioContent(url="http://example.com/audio.mp3"), + ImageContent(url=IMAGE_URL), + AudioContent(url=AUDIO_URL), ], }, {"role": AuthorRole.ASSISTANT, "content": "Assistant message"}, @@ -52,9 +126,7 @@ def test_phi4mm_template_with_text_and_image(): expected_output = ( "<|system|>\nSystem message<|end|>\n" - "<|user|>\nUser text message<|end|>\n" - "<|image_1|>\n" - "<|audio_1|>\n" + "<|user|>\n<|image_1|>\n<|audio_1|>\nUser text message<|end|>\n" "<|assistant|>\nAssistant message<|end|>\n" "<|assistant|>\n" ) @@ -62,6 +134,85 @@ def test_phi4mm_template_with_text_and_image(): assert phi4mm_template(history) == expected_output +def test_phi4mm_template_with_image_and_text(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [ImageContent(url=IMAGE_URL), TextContent(text="Describe this image.")], + }, + ] + ) + + expected_output = "<|user|>\n<|image_1|>\nDescribe this image.<|end|>\n<|assistant|>\n" + + assert phi4mm_template(history) == expected_output + + +def test_phi4mm_template_with_multiple_images(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [ + TextContent(text="Summarize the content of the images."), + ImageContent(url=IMAGE_URL), + ImageContent(url=IMAGE_URL), + ImageContent(url=IMAGE_URL), + ], + }, + ] + ) + + expected_output = ( + "<|user|>\n<|image_1|>\n<|image_2|>\n<|image_3|>\nSummarize the content of the images.<|end|>\n<|assistant|>\n" + ) + + assert phi4mm_template(history) == expected_output + + +def test_phi4mm_template_with_text_and_audio(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [TextContent(text="Transcribe the audio clip into text."), AudioContent(url=AUDIO_URL)], + }, + ] + ) + + expected_output = "<|user|>\n<|audio_1|>\nTranscribe the audio clip into text.<|end|>\n<|assistant|>\n" + + assert phi4mm_template(history) == expected_output + + +def test_phi4mm_template_with_audio_only(): + history = ChatHistory(messages=[{"role": AuthorRole.USER, "items": [AudioContent(url=AUDIO_URL)]}]) + + expected_output = "<|user|>\n<|audio_1|>\n<|end|>\n<|assistant|>\n" + + assert phi4mm_template(history) == expected_output + + +def test_phi4mm_template_puts_images_before_audio(): + history = ChatHistory( + messages=[ + { + "role": AuthorRole.USER, + "items": [ + AudioContent(url=AUDIO_URL), + ImageContent(url=IMAGE_URL), + ImageContent(url=IMAGE_URL), + ], + }, + ] + ) + + expected_output = "<|user|>\n<|image_1|>\n<|image_2|>\n<|audio_1|>\n<|end|>\n<|assistant|>\n" + + assert phi4mm_template(history) == expected_output + + def test_phi3_template_with_only_text(): history = ChatHistory(messages=[{"role": AuthorRole.USER, "items": [TextContent(text="User text message")]}])