diff --git a/python/semantic_kernel/template_engine/code_tokenizer.py b/python/semantic_kernel/template_engine/code_tokenizer.py index 7b4ebb9c4a2a..4d808ebe7e6e 100644 --- a/python/semantic_kernel/template_engine/code_tokenizer.py +++ b/python/semantic_kernel/template_engine/code_tokenizer.py @@ -48,6 +48,9 @@ def tokenize(text: str) -> list[Block]: space_separator_found = False skip_next_char = False next_char = "" + # True while reading the quoted value of a named argument (name='value'), + # so the token is flushed as a NamedArgBlock instead of a ValBlock. + named_arg_value = False blocks: list[Block] = [] for index, current_char in enumerate(text[:-1]): @@ -89,7 +92,11 @@ def tokenize(text: str) -> list[Block]: # When we reach the end of the value, we add the block if current_char == text_value_delimiter: - blocks.append(ValBlock(content="".join(current_token_content))) + if named_arg_value: + blocks.append(NamedArgBlock(content="".join(current_token_content))) + named_arg_value = False + else: + blocks.append(ValBlock(content="".join(current_token_content))) current_token_content.clear() current_token_type = None space_separator_found = False @@ -119,6 +126,21 @@ def tokenize(text: str) -> list[Block]: continue + # A quote directly after the "=" of a named argument starts a quoted + # value, e.g. "arg1='a b'". Entering value mode keeps spaces inside + # the quotes part of the same token instead of splitting it there. + if ( + current_token_type == BlockTypes.FUNCTION_ID + and current_char in (Symbols.DBL_QUOTE, Symbols.SGL_QUOTE) + and current_token_content + and current_token_content[-1] == Symbols.NAMED_ARG_BLOCK_SEPARATOR + ): + current_token_content.append(current_char) + current_token_type = BlockTypes.VALUE + text_value_delimiter = current_char + named_arg_value = True + continue + # If we're not inside a quoted value, and we're not processing a space current_token_content.append(current_char) @@ -143,7 +165,10 @@ def tokenize(text: str) -> list[Block]: current_token_content.append(next_char) if current_token_type == BlockTypes.VALUE: - blocks.append(ValBlock(content="".join(current_token_content))) + if named_arg_value: + blocks.append(NamedArgBlock(content="".join(current_token_content))) + else: + blocks.append(ValBlock(content="".join(current_token_content))) elif current_token_type == BlockTypes.VARIABLE: blocks.append(VarBlock(content="".join(current_token_content))) elif current_token_type == BlockTypes.FUNCTION_ID: diff --git a/python/tests/unit/template_engine/test_code_tokenizer.py b/python/tests/unit/template_engine/test_code_tokenizer.py index 42024c6120ad..bc14216a1156 100644 --- a/python/tests/unit/template_engine/test_code_tokenizer.py +++ b/python/tests/unit/template_engine/test_code_tokenizer.py @@ -128,3 +128,29 @@ def test_named_args(): assert blocks[1].content == '"direct"' assert blocks[2].content == "arg1=$arg1" assert blocks[3].content == 'arg2="arg2"' + + +def test_named_args_with_spaces_in_quoted_values(): + # Spaces inside the quoted value of a named argument used to split the + # token at the space, producing an invalid block instead of a NamedArgBlock. + template = "plugin.function arg1='value with spaces'" + blocks = CodeTokenizer.tokenize(template) + assert len(blocks) == 2 + assert blocks[0].content == "plugin.function" + assert blocks[0].type == BlockTypes.FUNCTION_ID + assert blocks[1].content == "arg1='value with spaces'" + assert blocks[1].type == BlockTypes.NAMED_ARG + + +@mark.parametrize( + "template, expected_content", + [ + ('plugin.function arg1="a b"', 'arg1="a b"'), + ("plugin.function 'positional value' arg2='a b'", "arg2='a b'"), + ("plugin.function arg1=$var arg2='a b'", "arg2='a b'"), + ], +) +def test_named_args_with_spaces_variants(template, expected_content): + blocks = CodeTokenizer.tokenize(template) + named_args = [block for block in blocks if block.type == BlockTypes.NAMED_ARG] + assert named_args[-1].content == expected_content