Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 44 additions & 17 deletions python/semantic_kernel/connectors/ai/onnx/utils.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Copyright (c) Microsoft. All rights reserved.
from enum import Enum

from semantic_kernel.contents import AuthorRole, ChatHistory, ImageContent, TextContent
from semantic_kernel.contents import AuthorRole, ChatHistory, ChatMessageContent, ImageContent, TextContent
from semantic_kernel.contents.audio_content import AudioContent
from semantic_kernel.exceptions import ServiceException, ServiceInvalidRequestError

Expand Down Expand Up @@ -104,12 +104,8 @@ def phi3v_template(history: ChatHistory) -> str:
if message.role == AuthorRole.SYSTEM:
phi3v_input += f"<|system|>\n{message.content}<|end|>\n"
if message.role == AuthorRole.USER:
for item in message.items:
if isinstance(item, TextContent):
phi3v_input += f"<|user|>\n{item.text}<|end|>\n"
if isinstance(item, ImageContent):
phi3v_input += f"<|image_{image_count + 1}|>\n"
image_count += 1
user_content, image_count, _ = _multimodal_user_content(message, image_count)
phi3v_input += f"<|user|>\n{user_content}<|end|>\n"
if message.role == AuthorRole.ASSISTANT:
phi3v_input += f"<|assistant|>\n{message.content}<|end|>\n"
phi3v_input += "<|assistant|>\n"
Expand All @@ -133,22 +129,53 @@ def phi4mm_template(history: ChatHistory) -> str:
if message.role == AuthorRole.SYSTEM:
phi4mm_input += f"<|system|>\n{message.content}<|end|>\n"
if message.role == AuthorRole.USER:
for item in message.items:
if isinstance(item, TextContent):
phi4mm_input += f"<|user|>\n{item.text}<|end|>\n"
# Currently only one image is supported in Onnx
if isinstance(item, ImageContent):
phi4mm_input += f"<|image_{image_count + 1}|>\n"
image_count += 1
if isinstance(item, AudioContent):
phi4mm_input += f"<|audio_{audio_count + 1}|>\n"
audio_count += 1
user_content, image_count, audio_count = _multimodal_user_content(
message, image_count, audio_count, include_audio=True
)
phi4mm_input += f"<|user|>\n{user_content}<|end|>\n"
if message.role == AuthorRole.ASSISTANT:
phi4mm_input += f"<|assistant|>\n{message.content}<|end|>\n"
phi4mm_input += "<|assistant|>\n"
return phi4mm_input


def _multimodal_user_content(
message: ChatMessageContent,
image_count: int,
audio_count: int = 0,
include_audio: bool = False,
) -> tuple[str, int, int]:
"""Builds the content of a single user turn for the Phi-3-vision and Phi-4-multimodal templates.

All content of the message goes into one user turn: the image placeholders first, then the audio
placeholders, each on its own line, then the text. This is the prompt format given in the model
cards and used by the onnxruntime-genai examples. Placeholders are numbered across the whole chat
history, starting at 1.

Args:
message (ChatMessageContent): The user message.
image_count (int): The number of images in the chat history before this message.
audio_count (int): The number of audio clips in the chat history before this message.
include_audio (bool): Whether to emit audio placeholders.

Returns:
tuple[str, int, int]: The user content and the updated image and audio counts.
"""
image_tags = ""
audio_tags = ""
texts: list[str] = []
for item in message.items:
if isinstance(item, TextContent):
texts.append(item.text)
elif isinstance(item, ImageContent):
image_count += 1
image_tags += f"<|image_{image_count}|>\n"
elif include_audio and isinstance(item, AudioContent):
audio_count += 1
audio_tags += f"<|audio_{audio_count}|>\n"
return image_tags + audio_tags + "\n".join(texts), image_count, audio_count


def gemma_template(history: ChatHistory) -> str:
"""Generates a formatted string for the Gemma model based on the provided chat history.

Expand Down
167 changes: 159 additions & 8 deletions python/tests/unit/connectors/ai/onnx/services/test_onnx_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,30 +10,104 @@
)
from semantic_kernel.contents import AudioContent, AuthorRole, ChatHistory, ImageContent, TextContent

IMAGE_URL = "http://example.com/image.png"
AUDIO_URL = "http://example.com/audio.mp3"


def test_phi3v_template_with_text_and_image():
history = ChatHistory(
messages=[
{"role": AuthorRole.SYSTEM, "content": "System message"},
{
"role": AuthorRole.USER,
"items": [TextContent(text="User text message"), ImageContent(url="http://example.com/image.png")],
"items": [TextContent(text="User text message"), ImageContent(url=IMAGE_URL)],
},
{"role": AuthorRole.ASSISTANT, "content": "Assistant message"},
]
)

expected_output = (
"<|system|>\nSystem message<|end|>\n"
"<|user|>\nUser text message<|end|>\n"
"<|image_1|>\n"
"<|user|>\n<|image_1|>\nUser text message<|end|>\n"
"<|assistant|>\nAssistant message<|end|>\n"
"<|assistant|>\n"
)

assert phi3v_template(history) == expected_output


def test_phi3v_template_with_image_and_text():
history = ChatHistory(
messages=[
{
"role": AuthorRole.USER,
"items": [ImageContent(url=IMAGE_URL), TextContent(text="Describe this image.")],
},
]
)

expected_output = "<|user|>\n<|image_1|>\nDescribe this image.<|end|>\n<|assistant|>\n"

assert phi3v_template(history) == expected_output


def test_phi3v_template_with_multiple_images():
history = ChatHistory(
messages=[
{
"role": AuthorRole.USER,
"items": [
ImageContent(url=IMAGE_URL),
ImageContent(url=IMAGE_URL),
TextContent(text="Compare these images."),
],
},
]
)

expected_output = "<|user|>\n<|image_1|>\n<|image_2|>\nCompare these images.<|end|>\n<|assistant|>\n"

assert phi3v_template(history) == expected_output


def test_phi3v_template_numbers_images_across_turns():
history = ChatHistory(
messages=[
{"role": AuthorRole.USER, "items": [ImageContent(url=IMAGE_URL), TextContent(text="First")]},
{"role": AuthorRole.ASSISTANT, "content": "Answer"},
{"role": AuthorRole.USER, "items": [TextContent(text="Second"), ImageContent(url=IMAGE_URL)]},
{"role": AuthorRole.ASSISTANT, "content": "Answer"},
{"role": AuthorRole.USER, "items": [TextContent(text="Third")]},
]
)

expected_output = (
"<|user|>\n<|image_1|>\nFirst<|end|>\n"
"<|assistant|>\nAnswer<|end|>\n"
"<|user|>\n<|image_2|>\nSecond<|end|>\n"
"<|assistant|>\nAnswer<|end|>\n"
"<|user|>\nThird<|end|>\n"
"<|assistant|>\n"
)

assert phi3v_template(history) == expected_output


def test_phi3v_template_merges_text_items_into_one_user_turn():
history = ChatHistory(
messages=[
{
"role": AuthorRole.USER,
"items": [TextContent(text="Line one"), ImageContent(url=IMAGE_URL), TextContent(text="Line two")],
},
]
)

expected_output = "<|user|>\n<|image_1|>\nLine one\nLine two<|end|>\n<|assistant|>\n"

assert phi3v_template(history) == expected_output


def test_phi4mm_template_with_text_and_image():
history = ChatHistory(
messages=[
Expand All @@ -42,8 +116,8 @@ def test_phi4mm_template_with_text_and_image():
"role": AuthorRole.USER,
"items": [
TextContent(text="User text message"),
ImageContent(url="http://example.com/image.png"),
AudioContent(url="http://example.com/audio.mp3"),
ImageContent(url=IMAGE_URL),
AudioContent(url=AUDIO_URL),
],
},
{"role": AuthorRole.ASSISTANT, "content": "Assistant message"},
Expand All @@ -52,16 +126,93 @@ def test_phi4mm_template_with_text_and_image():

expected_output = (
"<|system|>\nSystem message<|end|>\n"
"<|user|>\nUser text message<|end|>\n"
"<|image_1|>\n"
"<|audio_1|>\n"
"<|user|>\n<|image_1|>\n<|audio_1|>\nUser text message<|end|>\n"
"<|assistant|>\nAssistant message<|end|>\n"
"<|assistant|>\n"
)

assert phi4mm_template(history) == expected_output


def test_phi4mm_template_with_image_and_text():
history = ChatHistory(
messages=[
{
"role": AuthorRole.USER,
"items": [ImageContent(url=IMAGE_URL), TextContent(text="Describe this image.")],
},
]
)

expected_output = "<|user|>\n<|image_1|>\nDescribe this image.<|end|>\n<|assistant|>\n"

assert phi4mm_template(history) == expected_output


def test_phi4mm_template_with_multiple_images():
history = ChatHistory(
messages=[
{
"role": AuthorRole.USER,
"items": [
TextContent(text="Summarize the content of the images."),
ImageContent(url=IMAGE_URL),
ImageContent(url=IMAGE_URL),
ImageContent(url=IMAGE_URL),
],
},
]
)

expected_output = (
"<|user|>\n<|image_1|>\n<|image_2|>\n<|image_3|>\nSummarize the content of the images.<|end|>\n<|assistant|>\n"
)

assert phi4mm_template(history) == expected_output


def test_phi4mm_template_with_text_and_audio():
history = ChatHistory(
messages=[
{
"role": AuthorRole.USER,
"items": [TextContent(text="Transcribe the audio clip into text."), AudioContent(url=AUDIO_URL)],
},
]
)

expected_output = "<|user|>\n<|audio_1|>\nTranscribe the audio clip into text.<|end|>\n<|assistant|>\n"

assert phi4mm_template(history) == expected_output


def test_phi4mm_template_with_audio_only():
history = ChatHistory(messages=[{"role": AuthorRole.USER, "items": [AudioContent(url=AUDIO_URL)]}])

expected_output = "<|user|>\n<|audio_1|>\n<|end|>\n<|assistant|>\n"

assert phi4mm_template(history) == expected_output


def test_phi4mm_template_puts_images_before_audio():
history = ChatHistory(
messages=[
{
"role": AuthorRole.USER,
"items": [
AudioContent(url=AUDIO_URL),
ImageContent(url=IMAGE_URL),
ImageContent(url=IMAGE_URL),
],
},
]
)

expected_output = "<|user|>\n<|image_1|>\n<|image_2|>\n<|audio_1|>\n<|end|>\n<|assistant|>\n"

assert phi4mm_template(history) == expected_output


def test_phi3_template_with_only_text():
history = ChatHistory(messages=[{"role": AuthorRole.USER, "items": [TextContent(text="User text message")]}])

Expand Down
Loading