diff --git a/python/packages/redis/agent_framework_redis/_context_provider.py b/python/packages/redis/agent_framework_redis/_context_provider.py index 0770ae769a7..174731d02e1 100644 --- a/python/packages/redis/agent_framework_redis/_context_provider.py +++ b/python/packages/redis/agent_framework_redis/_context_provider.py @@ -39,6 +39,18 @@ _SCOPE_SCHEMA_VERSION = 1 _SCOPE_FIELDS = ("application_id", "agent_id", "user_id", "thread_id", "conversation_id") _SCOPE_TAG_SEPARATOR = "\x1f" +_KEY_HASH_VERSION = 1 + + +def _document_key_suffix(*, scope: dict[str, str | None], role: str, content: str) -> str: + """Stable hash of (scope, role, content) used as the document key suffix.""" + payload = json.dumps( + [_KEY_HASH_VERSION, [scope.get(name) or "" for name in _SCOPE_FIELDS], role, content], + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + _TAG_DEFAULTS = {"separator": ",", "case_sensitive": False, "withsuffixtrie": False} _TEXT_DEFAULTS = {"weight": 1.0, "no_stem": False} @@ -341,6 +353,31 @@ async def _existing_index(self) -> Any: redis_url=None if existing_client is not None else self.redis_url, ) + def _scope_values(self, session_id: str | None) -> dict[str, str | None]: + """Scope fields stored on, and used to key and filter, every memory document.""" + return { + "application_id": self.application_id, + "agent_id": self.agent_id, + "user_id": self.user_id, + "thread_id": session_id, + "conversation_id": session_id, + } + + def _document_redis_key(self, *, role: str, content: str, session_id: str | None) -> str: + """Full Redis key for a memory document, built from the real index prefix.""" + suffix = _document_key_suffix(scope=self._scope_values(session_id), role=role, content=content) + return self.redis_index.key(suffix) + + async def _existing_keys(self, keys: list[str]) -> set[str]: + """Return the subset of `keys` that already exist in Redis.""" + if not keys: + return set() + async with self.redis_index.client.pipeline(transaction=False) as pipe: + for key in keys: + pipe.exists(key) + results = await pipe.execute() + return {key for key, count in zip(keys, results, strict=True) if count} + async def _add( self, *, @@ -348,7 +385,32 @@ async def _add( session_id: str | None = None, metadata: dict[str, Any] | None = None, ) -> None: - """Inserts one or many documents with partition fields populated.""" + """Inserts one or many documents with partition fields populated. + + Documents are deduplicated by their scope fields, role, and content. + When the AG-UI client resends the whole conversation on every run, + already-stored messages are detected via a pipelined EXISTS check + and skipped, so they are not re-embedded or re-stored. + + Tradeoffs: + - The same text from the same role in one conversation is stored + once. Legitimate repeats ("yes", "thanks") collapse to one entry. + - Documents saved before this fix keep their old random keys, so an + existing conversation gets one extra copy of each message on the + first post-fix run and then stays stable. + - Content is normalized with .strip() before hashing, so whitespace + differences between the stored model text and the replayed AG-UI + text do not produce a second copy. + + Concurrency: the EXISTS check and the subsequent load are not atomic. + Two concurrent _add calls for the same message can both observe the + key absent, both embed, and both write. Because keys are + deterministic, the second write overwrites the first at the same + key, so the final state is always one document per (scope, role, + content). The cost of the race is at most one redundant embedding + call per concurrent duplicate; this is accepted over the complexity + of a SET NX reservation with crash recovery. + """ self._validate_filters() await self._ensure_index() docs = data if isinstance(data, list) else [data] @@ -356,14 +418,7 @@ async def _add( prepared: list[dict[str, Any]] = [] for doc in docs: d = dict(doc) - scope_values = { - "application_id": self.application_id, - "agent_id": self.agent_id, - "user_id": self.user_id, - "thread_id": session_id, - "conversation_id": session_id, - } - for name, value in scope_values.items(): + for name, value in self._scope_values(session_id).items(): d[name] = value or None d[_scope_token_field(name)] = _scope_token(name, value) if value else None if "content" not in d: @@ -372,23 +427,36 @@ async def _add( d.setdefault(self.vector_field_name, None) prepared.append(d) + docs_by_key: dict[str, dict[str, Any]] = {} + for d in prepared: + content = (d.get("content") or "").strip() + if not content: + continue + key = self._document_redis_key(role=d.get("role") or "", content=content, session_id=session_id) + docs_by_key.setdefault(key, d) + + existing = await self._existing_keys(list(docs_by_key)) + new_docs = {k: d for k, d in docs_by_key.items() if k not in existing} + if not new_docs: + return + keys_to_load = list(new_docs) + prepared_to_load = list(new_docs.values()) + if self.redis_vectorizer and self.vector_field_name: - text_list = [d["content"] for d in prepared] + text_list = [d["content"] for d in prepared_to_load] embeddings = await self.redis_vectorizer.aembed_many( # pyright: ignore[reportUnknownMemberType] text_list, batch_size=len(text_list) ) - for i, d in enumerate(prepared): + for i, d in enumerate(prepared_to_load): # aembed_many returns lists unless as_buffer is enabled. - vec = array_to_buffer( + d[self.vector_field_name] = array_to_buffer( embeddings[i], # pyright: ignore[reportArgumentType] dtype=self.redis_vectorizer.dtype, ) - field_name: str = self.vector_field_name - d[field_name] = vec # Redis hashes cannot encode None, including with non-validating borrowed indexes. - prepared = [{key: value for key, value in doc.items() if value is not None} for doc in prepared] - await self.redis_index.load(prepared) + prepared_to_load = [{k: v for k, v in d.items() if v is not None} for d in prepared_to_load] + await self.redis_index.load(prepared_to_load, keys=keys_to_load) async def _redis_search( self, @@ -410,13 +478,7 @@ async def _redis_search( raise IntegrationInvalidRequestException("text_search() requires non-empty text") num_results = max(int(num_results or 10), 1) - combined_filter = self._build_filter_from_dict({ - "application_id": self.application_id, - "agent_id": self.agent_id, - "user_id": self.user_id, - "thread_id": session_id, - "conversation_id": session_id, - }) + combined_filter = self._build_filter_from_dict(self._scope_values(session_id)) if filter_expression is not None: combined_filter = (combined_filter & filter_expression) if combined_filter else filter_expression diff --git a/python/packages/redis/tests/test_context_provider_edges.py b/python/packages/redis/tests/test_context_provider_edges.py index 1ae85cfe70a..3ce54fdcc7e 100644 --- a/python/packages/redis/tests/test_context_provider_edges.py +++ b/python/packages/redis/tests/test_context_provider_edges.py @@ -23,11 +23,28 @@ ) +def _make_pipeline_mock(**_: Any) -> MagicMock: + """Fresh pipeline mock per client.pipeline() call. + + A shared mock accumulates exists.call_count across _add calls, so a + second _add with nonempty candidates returns too many EXISTS results + and _existing_keys raises at its strict zip. + """ + pipe = MagicMock() + pipe.exists = MagicMock() + pipe.execute = AsyncMock(side_effect=lambda: [0] * pipe.exists.call_count) + pipe.__aenter__ = AsyncMock(return_value=pipe) + pipe.__aexit__ = AsyncMock(return_value=None) + return pipe + + @pytest.fixture def mock_index() -> AsyncMock: index = AsyncMock() client = MagicMock() + client.pipeline = MagicMock(side_effect=_make_pipeline_mock) index.client = client + index.key = MagicMock(side_effect=lambda suffix: f"context:{suffix}") index.create = AsyncMock() index.exists = AsyncMock(return_value=False) index.load = AsyncMock() diff --git a/python/packages/redis/tests/test_context_provider_integration.py b/python/packages/redis/tests/test_context_provider_integration.py index 1fc69ea3b09..7d8831866b1 100644 --- a/python/packages/redis/tests/test_context_provider_integration.py +++ b/python/packages/redis/tests/test_context_provider_integration.py @@ -239,3 +239,73 @@ async def test_live_legacy_index_fails_closed_until_recreated( assert await _provider(url, index_name, prefix, user_id="alice")._redis_search("re-ingested marker") == [] adjacent_record = await cast(Awaitable[dict[bytes, bytes]], client.hgetall(adjacent_key)) assert adjacent_record[b"content"] == b"foreign" + + +@pytest.mark.integration +async def test_live_deduplicates_replayed_conversation_messages( + live_redis: tuple[str, Redis, str, str], +) -> None: + """Re-sending the whole conversation does not duplicate documents. + + The AG-UI client sends all earlier messages on every run. Without + deduplication, a 15-fact conversation produces 240 documents and search + results are dominated by copies of early messages. With the fix, the + same conversation produces exactly one document per message and search + returns distinct results. + """ + url, client, index_name, prefix = live_redis + provider = _provider(url, index_name, prefix, user_id="user-1") + + facts = [ + "Our release day is Thursday.", + "We deploy to westus3.", + "Rollbacks need approval from Mira.", + "The on-call engineer this week is Priya.", + "Our database is Postgres 16.", + ] + + session_id = "repro-session" + sent_messages: list[dict[str, str]] = [] + for fact in facts: + sent_messages.append({"role": "user", "content": fact}) + await provider._add(data=list(sent_messages), session_id=session_id) + + encoded_prefix = prefix.encode() + keys = [ + key + async for key in client.scan_iter(_type="hash") + if (key if isinstance(key, bytes) else str(key).encode()).startswith(encoded_prefix) + ] + assert len(keys) == 5 + + stored_contents = [] + for key in keys: + key_str = key.decode() if isinstance(key, bytes) else str(key) + record = await cast(Awaitable[dict[bytes, bytes]], client.hgetall(key_str)) + stored_contents.append(record[b"content"].decode()) + assert stored_contents.count("We deploy to westus3.") == 1 + + results = await provider._redis_search("deploy", session_id=session_id, num_results=10) + assert len(results) >= 1 + assert results[0]["content"] == "We deploy to westus3." + + +@pytest.mark.integration +async def test_live_dedup_key_is_stable_across_runs( + live_redis: tuple[str, Redis, str, str], +) -> None: + """Storing the same message twice maps to the same Redis key.""" + url, client, index_name, prefix = live_redis + provider = _provider(url, index_name, prefix, user_id="user-1") + + session_id = "stable-key-session" + await provider._add(data={"role": "user", "content": "Remember tea"}, session_id=session_id) + await provider._add(data={"role": "user", "content": "Remember tea"}, session_id=session_id) + + encoded_prefix = prefix.encode() + keys = [ + key + async for key in client.scan_iter(_type="hash") + if (key if isinstance(key, bytes) else str(key).encode()).startswith(encoded_prefix) + ] + assert len(keys) == 1 diff --git a/python/packages/redis/tests/test_context_provider_sdk.py b/python/packages/redis/tests/test_context_provider_sdk.py index 47af6d523a5..af3a875affa 100644 --- a/python/packages/redis/tests/test_context_provider_sdk.py +++ b/python/packages/redis/tests/test_context_provider_sdk.py @@ -61,10 +61,14 @@ async def redis_client(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[redis.R await client.aclose() -def _provider(client: redis.Redis, *, borrowed: bool, **kwargs: Any) -> RedisContextProvider: +def _provider( + client: redis.Redis, + *, + borrowed: bool, + **kwargs: Any, +) -> RedisContextProvider: provider = RedisContextProvider(**kwargs) if borrowed: - # Borrowed indexes use RedisVL's default validate_on_load=False. index = AsyncSearchIndex.from_dict(provider.schema_dict, redis_client=client) provider = RedisContextProvider(redis_index=index, **kwargs) else: @@ -72,6 +76,7 @@ def _provider(client: redis.Redis, *, borrowed: bool, **kwargs: Any) -> RedisCon provider.redis_index.exists = AsyncMock(return_value=False) provider.redis_index.create = AsyncMock() object.__setattr__(provider, "_validate_schema_compatibility", AsyncMock()) + object.__setattr__(provider, "_existing_keys", AsyncMock(return_value=set())) return provider @@ -220,3 +225,224 @@ async def test_borrowed_index_retains_generated_vector_buffer( assert write.call_args.args[2]["embedding"] == array_to_buffer([1.0, 0.0], dtype="float32") assert len(_TransportConnection.packets) == 1 assert document == before + + +async def test_after_run_does_not_store_replayed_input_messages( + redis_client: redis.Redis, monkeypatch: pytest.MonkeyPatch +) -> None: + """Replayed input messages are not stored again.""" + provider = _provider(redis_client, borrowed=True, user_id="user-1") + write = _capture_hash_writes(monkeypatch) + session = AgentSession(session_id="session-1") + + first_message = Message(role="user", contents=["First fact"]) + await provider.after_run( + agent=MagicMock(), + session=session, + context=SessionContext(session_id=session.session_id, input_messages=[first_message]), + state={}, + ) + assert write.await_count == 1 + + first_key = provider._document_redis_key(role="user", content="First fact", session_id=session.session_id) + object.__setattr__(provider, "_existing_keys", AsyncMock(return_value={first_key})) + second_message = Message(role="user", contents=["Second fact"]) + await provider.after_run( + agent=MagicMock(), + session=session, + context=SessionContext( + session_id=session.session_id, + input_messages=[first_message, second_message], + ), + state={}, + ) + + assert write.await_count == 2 + written_contents = [call.args[2]["content"] for call in write.call_args_list] + assert written_contents.count("First fact") == 1 + assert written_contents.count("Second fact") == 1 + + +async def test_add_skips_embedding_for_already_stored_documents( + redis_client: redis.Redis, monkeypatch: pytest.MonkeyPatch +) -> None: + """Already-stored documents are skipped before the embedding call.""" + vectorizer = MagicMock(spec=BaseVectorizer) + vectorizer.dims = 2 + vectorizer.dtype = "float32" + vectorizer.aembed_many = AsyncMock(return_value=[[1.0, 0.0]]) + provider = _provider( + redis_client, + borrowed=True, + user_id="user-1", + vector_field_name="embedding", + redis_vectorizer=vectorizer, + ) + write = _capture_hash_writes(monkeypatch) + + document = {"role": "user", "content": "Remember tea"} + key = provider._document_redis_key(role="user", content="Remember tea", session_id="session-1") + object.__setattr__(provider, "_existing_keys", AsyncMock(return_value={key})) + + await provider._add(data=document, session_id="session-1") + + vectorizer.aembed_many.assert_not_awaited() + write.assert_not_awaited() + + +async def test_add_deduplicates_within_a_single_batch( + redis_client: redis.Redis, monkeypatch: pytest.MonkeyPatch +) -> None: + """The same (scope, role, content) appearing twice in one batch is stored once.""" + provider = _provider(redis_client, borrowed=True, user_id="user-1") + write = _capture_hash_writes(monkeypatch) + + documents = [ + {"role": "user", "content": "Duplicate fact"}, + {"role": "user", "content": "Duplicate fact"}, + ] + await provider._add(data=documents, session_id="session-1") + + write.assert_awaited_once() + + +async def test_add_calls_existing_keys_with_candidate_keys( + redis_client: redis.Redis, monkeypatch: pytest.MonkeyPatch +) -> None: + """_add checks Redis for existing keys before loading.""" + provider = _provider(redis_client, borrowed=True, user_id="user-1") + existing_keys_mock = AsyncMock(return_value=set()) + object.__setattr__(provider, "_existing_keys", existing_keys_mock) + _capture_hash_writes(monkeypatch) + + document = {"role": "user", "content": "Remember tea"} + await provider._add(data=document, session_id="session-1") + + existing_keys_mock.assert_awaited_once() + candidate_keys = existing_keys_mock.call_args.args[0] + assert len(candidate_keys) == 1 + expected_key = provider._document_redis_key(role="user", content="Remember tea", session_id="session-1") + assert candidate_keys[0] == expected_key + + +async def test_add_collapses_messages_with_same_content_but_different_message_ids( + redis_client: redis.Redis, monkeypatch: pytest.MonkeyPatch +) -> None: + """Messages with different message_ids but same scope/role/content share a key. + + The assistant reply comes back with a different id than the one stored + , so the key must not depend on message_id. + """ + provider = _provider(redis_client, borrowed=True, user_id="user-1") + write = _capture_hash_writes(monkeypatch) + + documents = [ + {"role": "assistant", "content": "We deploy to westus3.", "message_id": "chatcmpl-abc"}, + {"role": "assistant", "content": "We deploy to westus3.", "message_id": "agui-stream-xyz"}, + ] + await provider._add(data=documents, session_id="session-1") + + write.assert_awaited_once() + + +async def test_document_key_differs_across_scopes_roles_or_content( + redis_client: redis.Redis, +) -> None: + """The document key changes when scope, role, or content changes.""" + provider = _provider(redis_client, borrowed=True, user_id="user-1", application_id="app-1") + base = provider._document_redis_key(role="user", content="We deploy to westus3.", session_id="session-1") + different_session = provider._document_redis_key( + role="user", content="We deploy to westus3.", session_id="session-2" + ) + different_role = provider._document_redis_key( + role="assistant", content="We deploy to westus3.", session_id="session-1" + ) + different_content = provider._document_redis_key( + role="user", content="We deploy to eastus2.", session_id="session-1" + ) + assert len({base, different_session, different_role, different_content}) == 4 + + +async def test_add_skips_whitespace_only_content(redis_client: redis.Redis, monkeypatch: pytest.MonkeyPatch) -> None: + """Whitespace-only content is skipped before hashing. + + Tool-call messages often have content == "" — without skipping, all of + them with the same role would collapse to one key. + """ + provider = _provider(redis_client, borrowed=True, user_id="user-1") + write = _capture_hash_writes(monkeypatch) + + documents = [ + {"role": "user", "content": " "}, + {"role": "assistant", "content": ""}, + ] + await provider._add(data=documents, session_id="session-1") + + write.assert_not_awaited() + + +async def test_add_strips_content_before_keying(redis_client: redis.Redis, monkeypatch: pytest.MonkeyPatch) -> None: + """Whitespace differences do not produce a second copy. + + The replayed assistant text from AG-UI may differ slightly in + whitespace from the stored model text — .strip() prevents a second copy. + """ + provider = _provider(redis_client, borrowed=True, user_id="user-1") + write = _capture_hash_writes(monkeypatch) + + documents = [ + {"role": "user", "content": "hi"}, + {"role": "user", "content": "hi "}, + ] + await provider._add(data=documents, session_id="session-1") + + write.assert_awaited_once() + + +async def test_add_with_mixed_existing_keys_embeds_only_missing( + redis_client: redis.Redis, monkeypatch: pytest.MonkeyPatch +) -> None: + """Only documents whose keys are absent get embedded and loaded. + + Exercises the real _existing_keys pipeline with mixed EXISTS results: + one key exists, one does not. Only the missing document should reach + the vectorizer and the loader. + """ + vectorizer = MagicMock(spec=BaseVectorizer) + vectorizer.dims = 2 + vectorizer.dtype = "float32" + vectorizer.aembed_many = AsyncMock(return_value=[[1.0, 0.0]]) + + provider = _provider( + redis_client, + borrowed=True, + user_id="user-1", + vector_field_name="embedding", + redis_vectorizer=vectorizer, + ) + provider.__dict__.pop("_existing_keys", None) + provider.redis_index.load = AsyncMock() + + def make_pipeline(**_: Any) -> MagicMock: + pipe = MagicMock() + pipe.exists = MagicMock() + pipe.execute = AsyncMock(side_effect=lambda: [1, 0][: pipe.exists.call_count]) + pipe.__aenter__ = AsyncMock(return_value=pipe) + pipe.__aexit__ = AsyncMock(return_value=None) + return pipe + + monkeypatch.setattr(redis_client, "pipeline", MagicMock(side_effect=make_pipeline)) + + documents = [ + {"role": "user", "content": "Already stored"}, + {"role": "user", "content": "New message"}, + ] + await provider._add(data=documents, session_id="session-1") + + vectorizer.aembed_many.assert_awaited_once_with(["New message"], batch_size=1) + provider.redis_index.load.assert_awaited_once() + load_args = provider.redis_index.load.await_args + assert load_args is not None + loaded_docs = load_args.args[0] + assert len(loaded_docs) == 1 + assert loaded_docs[0]["content"] == "New message" diff --git a/python/packages/redis/tests/test_providers.py b/python/packages/redis/tests/test_providers.py index ec66992d7f9..03921fe08f1 100644 --- a/python/packages/redis/tests/test_providers.py +++ b/python/packages/redis/tests/test_providers.py @@ -33,12 +33,29 @@ async def test_empty_history_save_marks_redis_used(mock_redis_client: MagicMock) mark_feature_used.assert_called_once_with(FeatureIndex.REDIS) +def _make_pipeline_mock(**_: Any) -> MagicMock: + """Fresh pipeline mock per client.pipeline() call. + + A shared mock accumulates exists.call_count across _add calls, so a + second _add with nonempty candidates returns too many EXISTS results + and _existing_keys raises at its strict zip. + """ + pipe = MagicMock() + pipe.exists = MagicMock() + pipe.execute = AsyncMock(side_effect=lambda: [0] * pipe.exists.call_count) + pipe.__aenter__ = AsyncMock(return_value=pipe) + pipe.__aexit__ = AsyncMock(return_value=None) + return pipe + + @pytest.fixture def mock_index() -> AsyncMock: idx = AsyncMock() client = MagicMock() client.aclose = AsyncMock() + client.pipeline = MagicMock(side_effect=_make_pipeline_mock) idx.client = client + idx.key = MagicMock(side_effect=lambda suffix: f"context:{suffix}") idx.create = AsyncMock() idx.load = AsyncMock() idx.query = AsyncMock(return_value=[])