Python: keep quoted named-arg values as single tokens in CodeTokenizer

python/semantic_kernel/template_engine/code_tokenizer.py

@@ -48,6 +48,9 @@ def tokenize(text: str) -> list[Block]:
         space_separator_found = False
         skip_next_char = False
         next_char = ""
+        # True while reading the quoted value of a named argument (name='value'),
+        # so the token is flushed as a NamedArgBlock instead of a ValBlock.
+        named_arg_value = False
         blocks: list[Block] = []
 
         for index, current_char in enumerate(text[:-1]):
@@ -89,7 +92,11 @@ def tokenize(text: str) -> list[Block]:
 
                 # When we reach the end of the value, we add the block
                 if current_char == text_value_delimiter:
-                    blocks.append(ValBlock(content="".join(current_token_content)))
+                    if named_arg_value:
+                        blocks.append(NamedArgBlock(content="".join(current_token_content)))
+                        named_arg_value = False
+                    else:
+                        blocks.append(ValBlock(content="".join(current_token_content)))
                     current_token_content.clear()
                     current_token_type = None
                     space_separator_found = False
@@ -119,6 +126,21 @@ def tokenize(text: str) -> list[Block]:
 
                 continue
 
+            # A quote directly after the "=" of a named argument starts a quoted
+            # value, e.g. "arg1='a b'". Entering value mode keeps spaces inside
+            # the quotes part of the same token instead of splitting it there.
+            if (
+                current_token_type == BlockTypes.FUNCTION_ID
+                and current_char in (Symbols.DBL_QUOTE, Symbols.SGL_QUOTE)
+                and current_token_content
+                and current_token_content[-1] == Symbols.NAMED_ARG_BLOCK_SEPARATOR
+            ):
+                current_token_content.append(current_char)
+                current_token_type = BlockTypes.VALUE
+                text_value_delimiter = current_char
+                named_arg_value = True
+                continue
+
             # If we're not inside a quoted value, and we're not processing a space
             current_token_content.append(current_char)
 
@@ -143,7 +165,10 @@ def tokenize(text: str) -> list[Block]:
         current_token_content.append(next_char)
 
         if current_token_type == BlockTypes.VALUE:
-            blocks.append(ValBlock(content="".join(current_token_content)))
+            if named_arg_value:
+                blocks.append(NamedArgBlock(content="".join(current_token_content)))
+            else:
+                blocks.append(ValBlock(content="".join(current_token_content)))
         elif current_token_type == BlockTypes.VARIABLE:
             blocks.append(VarBlock(content="".join(current_token_content)))
         elif current_token_type == BlockTypes.FUNCTION_ID:

python/tests/unit/template_engine/test_code_tokenizer.py

@@ -128,3 +128,29 @@ def test_named_args():
     assert blocks[1].content == '"direct"'
     assert blocks[2].content == "arg1=$arg1"
     assert blocks[3].content == 'arg2="arg2"'
+
+
+def test_named_args_with_spaces_in_quoted_values():
+    # Spaces inside the quoted value of a named argument used to split the
+    # token at the space, producing an invalid block instead of a NamedArgBlock.
+    template = "plugin.function arg1='value with spaces'"
+    blocks = CodeTokenizer.tokenize(template)
+    assert len(blocks) == 2
+    assert blocks[0].content == "plugin.function"
+    assert blocks[0].type == BlockTypes.FUNCTION_ID
+    assert blocks[1].content == "arg1='value with spaces'"
+    assert blocks[1].type == BlockTypes.NAMED_ARG
+
+
+@mark.parametrize(
+    "template, expected_content",
+    [
+        ('plugin.function arg1="a b"', 'arg1="a b"'),
+        ("plugin.function 'positional value' arg2='a b'", "arg2='a b'"),
+        ("plugin.function arg1=$var arg2='a b'", "arg2='a b'"),
+    ],
+)
+def test_named_args_with_spaces_variants(template, expected_content):
+    blocks = CodeTokenizer.tokenize(template)
+    named_args = [block for block in blocks if block.type == BlockTypes.NAMED_ARG]
+    assert named_args[-1].content == expected_content