From 37bb4456e352baf5b2324bd6e07ada0cf5f6d1a7 Mon Sep 17 00:00:00 2001 From: wxhking <186391338+wxhking@users.noreply.github.com> Date: Tue, 3 Nov 2026 21:10:32 +0800 Subject: [PATCH 1/2] fix(redis): escape RediSearch query syntax in memory search text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RedisContextProvider passed the user's message straight into TextQuery / AggregateHybridQuery as the search text. RediSearch reads several punctuation marks as query syntax there — '|' unions terms, '?' is a fuzzy marker, backticks quote exact phrases — so ordinary messages like 'ps aux | grep python' or 'Why ?' reached the engine as malformed grammar (e.g. '@content:(ps | aux | | | grep | python)') and raised a syntax error that failed the whole agent run; the model was never called. Escape the search text with a TokenEscaper configured for the RediSearch syntax characters before building either query. Spaces stay unescaped, so the terms keep tokenizing and BM25 scoring works as before. Tag filter values were already escaped through _vector_store's escaper; only the free-text path was exposed. Fixes #10002 --- .../_context_provider.py | 12 +++- .../tests/test_context_provider_edges.py | 62 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/python/packages/redis/agent_framework_redis/_context_provider.py b/python/packages/redis/agent_framework_redis/_context_provider.py index 0770ae769a7..ee4ef5e0230 100644 --- a/python/packages/redis/agent_framework_redis/_context_provider.py +++ b/python/packages/redis/agent_framework_redis/_context_provider.py @@ -10,6 +10,7 @@ import hashlib import json +import re import sys from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self @@ -24,6 +25,7 @@ from redisvl.query import AggregateHybridQuery, FilterQuery, TextQuery from redisvl.query.filter import FilterExpression from redisvl.redis.utils import array_to_buffer +from redisvl.utils.token_escaper import TokenEscaper from redisvl.utils.vectorize import BaseVectorizer from ._feature_usage import FeatureIndex @@ -43,6 +45,14 @@ _TAG_DEFAULTS = {"separator": ",", "case_sensitive": False, "withsuffixtrie": False} _TEXT_DEFAULTS = {"weight": 1.0, "no_stem": False} +# RediSearch reads several punctuation marks as query syntax — `|` unions terms, `?` is a fuzzy +# marker, backticks quote exact phrases — so a user message like "ps aux | grep python" or +# "Why ?" reaches the engine as a malformed query and raises a syntax error that fails the whole +# agent run. The search text is plain prose, not a query: escape the syntax characters before it +# reaches TextQuery / AggregateHybridQuery. Unlike the default TokenEscaper set (built for tag +# filter values), spaces stay unescaped so the terms keep tokenizing normally. +_SEARCH_TEXT_ESCAPER = TokenEscaper(re.compile(r"[,.<>{}\[\]\\\"':;!@#$%^&*()\-+=~/|?`]")) + def _scope_token_field(name: str) -> str: if name not in _SCOPE_FIELDS: @@ -405,7 +415,7 @@ async def _redis_search( await self._ensure_index() self._validate_filters() - q = (text or "").strip() + q = _SEARCH_TEXT_ESCAPER.escape((text or "").strip()) if not q: raise IntegrationInvalidRequestException("text_search() requires non-empty text") num_results = max(int(num_results or 10), 1) diff --git a/python/packages/redis/tests/test_context_provider_edges.py b/python/packages/redis/tests/test_context_provider_edges.py index 1ae85cfe70a..368642e4edf 100644 --- a/python/packages/redis/tests/test_context_provider_edges.py +++ b/python/packages/redis/tests/test_context_provider_edges.py @@ -353,6 +353,68 @@ async def test_redis_search_combines_explicit_filter_expression( mock_index.query.assert_awaited_once() +@pytest.mark.parametrize( + ("message", "broken_shape", "escaped"), + [ + ("Why ?", "| ?)", r"\?"), + ("ps aux | grep python", "| |)", r"\|"), + ("Is it A | B?", "| |)", r"\|"), + ("Run ` ls ` please", "| ` |", r"\`"), + ("| name | age |", "| |)", r"\|"), + ], +) +async def test_redis_search_escapes_query_syntax_in_user_text( + message: str, + broken_shape: str, + escaped: str, + mock_index: AsyncMock, + patch_index: MagicMock, # noqa: ARG001 +) -> None: + """User prose reaches RediSearch as escaped terms, not as query syntax (#10002). + + A bare `|`, `?` or backtick in a message previously turned the generated query + string into malformed grammar (`@content:(ps | aux | | | grep | python)`), and + the syntax error failed the whole agent run. + """ + provider = RedisContextProvider(source_id="ctx", user_id="user-1") + + with patch.object(provider, "_ensure_index", AsyncMock()): + await provider._redis_search(text=message) + + query = mock_index.query.await_args.args[0] + query_string = query.query_string() + assert broken_shape not in query_string + assert escaped in query_string + + +async def test_redis_search_escapes_text_for_hybrid_query( + mock_index: AsyncMock, + patch_index: MagicMock, # noqa: ARG001 +) -> None: + """The hybrid vector path gets the same escaped text.""" + mock_vectorizer = MagicMock(spec=BaseVectorizer) + mock_vectorizer.dims = 128 + mock_vectorizer.dtype = "float32" + mock_vectorizer.aembed = AsyncMock(return_value=[0.1] * 128) + + provider = RedisContextProvider( + source_id="ctx", + user_id="u1", + redis_vectorizer=mock_vectorizer, + vector_field_name="embedding", + ) + + with ( + patch.object(provider, "_ensure_index", AsyncMock()), + patch("agent_framework_redis._context_provider.AggregateHybridQuery") as hybrid_query, + ): + hybrid_query.return_value = MagicMock() + await provider._redis_search(text="ps aux | grep python") + + assert hybrid_query.call_args.kwargs["text"] == r"ps aux \| grep python" + mock_index.query.assert_awaited_once() + + async def test_search_all_collects_paginated_batches( mock_index: AsyncMock, patch_index: MagicMock, # noqa: ARG001 From d12730eb84129aa28bff83d1c522b576d2ecbb89 Mon Sep 17 00:00:00 2001 From: wxhking <186391338+wxhking@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:33:31 +0800 Subject: [PATCH 2/2] fix(redis): drop redis search syntax chars instead of double escaping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit redisvl's TextQuery escapes every term itself: _tokenize_and_escape_query runs the default TokenEscaper per token, and that escaper also escapes the backslash a hand-escaped term already carries. The pre-escaped '|' from the TokenEscaper built here therefore landed as a literal backslash-pipe that was re-escaped again — the produced query still carried the bare '|', so 'ps aux | grep python' stayed broken, and tokens like 'v1.2' picked up a backslash that changed BM25 matching. Instead of escaping, strip the syntax characters redisvl's escaper does not cover ('|', '?', backtick) and rejoin the remaining tokens as plain terms; TextQuery then tokenizes and escapes each of them again as designed. Suggested-by: Aman Agnihotri <163747435+its-amann@users.noreply.github.com> --- .../_context_provider.py | 17 +++---- .../tests/test_context_provider_edges.py | 46 ++++++++++++------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/python/packages/redis/agent_framework_redis/_context_provider.py b/python/packages/redis/agent_framework_redis/_context_provider.py index ee4ef5e0230..6f5baa661cb 100644 --- a/python/packages/redis/agent_framework_redis/_context_provider.py +++ b/python/packages/redis/agent_framework_redis/_context_provider.py @@ -25,7 +25,6 @@ from redisvl.query import AggregateHybridQuery, FilterQuery, TextQuery from redisvl.query.filter import FilterExpression from redisvl.redis.utils import array_to_buffer -from redisvl.utils.token_escaper import TokenEscaper from redisvl.utils.vectorize import BaseVectorizer from ._feature_usage import FeatureIndex @@ -45,14 +44,6 @@ _TAG_DEFAULTS = {"separator": ",", "case_sensitive": False, "withsuffixtrie": False} _TEXT_DEFAULTS = {"weight": 1.0, "no_stem": False} -# RediSearch reads several punctuation marks as query syntax — `|` unions terms, `?` is a fuzzy -# marker, backticks quote exact phrases — so a user message like "ps aux | grep python" or -# "Why ?" reaches the engine as a malformed query and raises a syntax error that fails the whole -# agent run. The search text is plain prose, not a query: escape the syntax characters before it -# reaches TextQuery / AggregateHybridQuery. Unlike the default TokenEscaper set (built for tag -# filter values), spaces stay unescaped so the terms keep tokenizing normally. -_SEARCH_TEXT_ESCAPER = TokenEscaper(re.compile(r"[,.<>{}\[\]\\\"':;!@#$%^&*()\-+=~/|?`]")) - def _scope_token_field(name: str) -> str: if name not in _SCOPE_FIELDS: @@ -415,9 +406,13 @@ async def _redis_search( await self._ensure_index() self._validate_filters() - q = _SEARCH_TEXT_ESCAPER.escape((text or "").strip()) - if not q: + if not (text or "").strip(): raise IntegrationInvalidRequestException("text_search() requires non-empty text") + # TextQuery escapes each term itself, so a backslash added here gets escaped again + # and the char stays bare. Drop the syntax chars its escaper misses instead. + q = " ".join(re.sub(r"[|?`]", " ", text).split()) + if not q: + return [] num_results = max(int(num_results or 10), 1) combined_filter = self._build_filter_from_dict({ diff --git a/python/packages/redis/tests/test_context_provider_edges.py b/python/packages/redis/tests/test_context_provider_edges.py index 368642e4edf..e68bf630bfc 100644 --- a/python/packages/redis/tests/test_context_provider_edges.py +++ b/python/packages/redis/tests/test_context_provider_edges.py @@ -354,27 +354,28 @@ async def test_redis_search_combines_explicit_filter_expression( @pytest.mark.parametrize( - ("message", "broken_shape", "escaped"), + ("message", "terms"), [ - ("Why ?", "| ?)", r"\?"), - ("ps aux | grep python", "| |)", r"\|"), - ("Is it A | B?", "| |)", r"\|"), - ("Run ` ls ` please", "| ` |", r"\`"), - ("| name | age |", "| |)", r"\|"), + ("Why ?", "(why)"), + ("ps aux | grep python", "(ps | aux | grep | python)"), + ("Is it A | B?", "(is | it | a | b)"), + ("Run ` ls ` please", "(run | ls | please)"), + ("| name | age |", "(name | age)"), + ("see e.g. v1.2", r"(see | e\.g\. | v1\.2)"), ], ) -async def test_redis_search_escapes_query_syntax_in_user_text( +async def test_redis_search_drops_query_syntax_in_user_text( message: str, - broken_shape: str, - escaped: str, + terms: str, mock_index: AsyncMock, patch_index: MagicMock, # noqa: ARG001 ) -> None: - """User prose reaches RediSearch as escaped terms, not as query syntax (#10002). + """User prose reaches RediSearch as plain terms, not as query syntax (#10002). A bare `|`, `?` or backtick in a message previously turned the generated query string into malformed grammar (`@content:(ps | aux | | | grep | python)`), and - the syntax error failed the whole agent run. + the syntax error failed the whole agent run. redisvl escapes every term itself, + so nothing here may add a backslash of its own (it would be escaped again). """ provider = RedisContextProvider(source_id="ctx", user_id="user-1") @@ -383,15 +384,28 @@ async def test_redis_search_escapes_query_syntax_in_user_text( query = mock_index.query.await_args.args[0] query_string = query.query_string() - assert broken_shape not in query_string - assert escaped in query_string + assert f"@content:{terms}" in query_string + assert "\\\\" not in query_string -async def test_redis_search_escapes_text_for_hybrid_query( +async def test_redis_search_only_syntax_chars_returns_no_results( mock_index: AsyncMock, patch_index: MagicMock, # noqa: ARG001 ) -> None: - """The hybrid vector path gets the same escaped text.""" + """A message made only of query syntax has no terms to search for.""" + provider = RedisContextProvider(source_id="ctx", user_id="user-1") + + with patch.object(provider, "_ensure_index", AsyncMock()): + assert await provider._redis_search(text=" ? | ` ") == [] + + mock_index.query.assert_not_awaited() + + +async def test_redis_search_drops_syntax_for_hybrid_query( + mock_index: AsyncMock, + patch_index: MagicMock, # noqa: ARG001 +) -> None: + """The hybrid vector path gets the same cleaned text.""" mock_vectorizer = MagicMock(spec=BaseVectorizer) mock_vectorizer.dims = 128 mock_vectorizer.dtype = "float32" @@ -411,7 +425,7 @@ async def test_redis_search_escapes_text_for_hybrid_query( hybrid_query.return_value = MagicMock() await provider._redis_search(text="ps aux | grep python") - assert hybrid_query.call_args.kwargs["text"] == r"ps aux \| grep python" + assert hybrid_query.call_args.kwargs["text"] == "ps aux grep python" mock_index.query.assert_awaited_once()