From 8d4eb109881a1379292f8b94a8adbb846c6a6fe4 Mon Sep 17 00:00:00 2001 From: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:18:33 +0530 Subject: [PATCH 1/8] Python: give memory topic files an injective filename stem Distinct topics can share a readable slug, so a slug-only stem let the second topic silently overwrite the first. The stem now appends a digest of the normalized topic, and the storage layer keeps one authoritative record: lookups validate every candidate against the record it actually holds instead of accepting a filename prefix, so a missing topic can never read, merge into, or delete a longer slug; rewriting a topic that lived at a legacy slug-only file retires the legacy file; index pointer links and provider lookups use the persisted stem; and an explicit slug override is persisted in the record and stays findable by that slug, without letting lookups for other names bleed into the overridden record. Signed-off-by: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> --- .../core/agent_framework/_harness/_memory.py | 120 ++++++++++++-- .../core/tests/core/test_harness_memory.py | 154 ++++++++++++++++++ 2 files changed, 264 insertions(+), 10 deletions(-) diff --git a/python/packages/core/agent_framework/_harness/_memory.py b/python/packages/core/agent_framework/_harness/_memory.py index c3f02f883b5..ea6ad95f244 100644 --- a/python/packages/core/agent_framework/_harness/_memory.py +++ b/python/packages/core/agent_framework/_harness/_memory.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import hashlib import json import logging import re @@ -129,6 +130,22 @@ def _slugify_topic(topic: str) -> str: return slug or "memory-topic" +def _topic_stem(topic: str, *, slug_override: str | None = None) -> str: + """Return the injective filename stem for a topic. + + Distinct topics can share a readable slug: ``C ++`` and ``C #`` both slugify + to ``c``, so a slug-only stem let the second topic silently overwrite the + first topic's memories. The stem keeps the readable slug for humans and + appends a short digest of the normalized topic, so distinct normalized + topics never share a file (mirroring how ``_storage_key_segment`` keeps + byte-distinct identifiers in separate storage locations). + """ + normalized = _normalize_topic(topic) + slug = _slugify_topic(slug_override if slug_override is not None else normalized) + digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:8] + return f"{slug}-{digest}" + + def _timestamp(value: datetime | None = None) -> str: current = value or datetime.now(timezone.utc) return current.replace(microsecond=0).isoformat() @@ -302,7 +319,9 @@ def from_topic_record(cls, record: MemoryTopicRecord) -> MemoryIndexEntry: """ return cls( topic=record.topic, - slug=record.slug, + # The persisted stem, not the readable slug: distinct topics can + # share a slug, and an index link must point at the real file. + slug=record.storage_stem, summary=_coerce_summary(record.summary, record.memories), updated_at=record.updated_at, ) @@ -373,6 +392,11 @@ def __init__( self.updated_at = updated_at self.session_ids = _dedupe_strings(session_ids or []) + @property + def storage_stem(self) -> str: + """Filename stem this record persists under (slug or explicit override plus digest).""" + return _topic_stem(self.topic, slug_override=self.slug) + def to_dict(self) -> dict[str, Any]: """Serialize the topic record for tool output. @@ -449,9 +473,13 @@ def to_markdown(self) -> str: memory_lines = [f"- {_escape_markdown_line(memory)}" for memory in self.memories] or [ f"- {DEFAULT_MEMORY_NO_TOPICS_TEXT[2:]}" ] + header_lines = [f"# {self.topic}", ""] + if self.slug != _slugify_topic(self.topic): + # Persist an explicit slug override so later lookups by that slug, + # and the index stem, still resolve after a reload. + header_lines += [f"Slug: {self.slug}", ""] return "\n".join([ - f"# {self.topic}", - "", + *header_lines, f"Updated: {self.updated_at}", f"Sessions: {session_line}", "", @@ -479,6 +507,7 @@ def from_markdown(cls, markdown: str, *, fallback_topic: str | None = None) -> M ValueError: If the markdown cannot be parsed into the expected format. """ topic = fallback_topic + slug: str | None = None updated_at = _timestamp() session_ids: list[str] = [] summary_lines: list[str] = [] @@ -492,6 +521,9 @@ def from_markdown(cls, markdown: str, *, fallback_topic: str | None = None) -> M topic = stripped[2:].strip() current_section = None continue + if current_section is None and stripped.startswith("Slug: "): + slug = stripped.removeprefix("Slug: ").strip() + continue if current_section is None and stripped.startswith("Updated: "): updated_at = stripped.removeprefix("Updated: ").strip() continue @@ -522,6 +554,7 @@ def from_markdown(cls, markdown: str, *, fallback_topic: str | None = None) -> M raise ValueError("Memory topic markdown is missing a '# ' heading.") return cls( topic=topic, + slug=slug, summary="\n".join(summary_lines).strip(), memories=memories, updated_at=updated_at, @@ -763,9 +796,67 @@ def _get_index_path(self, session: AgentSession, *, source_id: str) -> Path: def _get_state_path(self, session: AgentSession, *, source_id: str) -> Path: return self._get_memory_root(session, source_id=source_id) / self.state_file_name - def _topic_path(self, session: AgentSession, *, source_id: str, topic: str) -> Path: + def _topic_path( + self, session: AgentSession, *, source_id: str, topic: str, slug_override: str | None = None + ) -> Path: + return ( + self._get_topics_directory(session, source_id=source_id) + / f"{_topic_stem(topic, slug_override=slug_override)}.md" + ) + + def _legacy_topic_path(self, session: AgentSession, *, source_id: str, topic: str) -> Path: + """Filename stem used before the digest suffix (kept for reading old stores).""" return self._get_topics_directory(session, source_id=source_id) / f"{_slugify_topic(topic)}.md" + def _existing_topic_path(self, session: AgentSession, *, source_id: str, topic: str) -> Path | None: + """Return the topic file to read, or ``None`` when no file matches. + + Lookups resolve in order: the digest stem of the given name (the exact + file ``write_topic`` uses for that topic), a legacy slug-only file from + stores written before the digest suffix, then any file whose stem starts + with the readable slug followed by a dash, which serves lookups made by + slug or by a topic that normalizes differently than its stored spelling. + When several topics share a slug, a slug-only lookup is ambiguous by + nature; the file whose recorded topic matches the lookup wins, else the + first stem in sorted order keeps the choice deterministic. + """ + topics_dir = self._get_topics_directory(session, source_id=source_id) + if not topics_dir.is_dir(): + return None + try: + wanted = _normalize_topic(topic) + except ValueError: + return None + # 1. The digest stem of the given name: the exact file write_topic uses. + topic_path = self._topic_path(session, source_id=source_id, topic=topic) + if topic_path.exists(): + return topic_path + # 2. An exact filename match: a legacy slug-only file from stores written + # before the digest suffix, or a caller passing a persisted stem. + exact_path = topics_dir / f"{_slugify_topic(topic)}.md" + if exact_path.exists(): + return exact_path + # 3. Digest-suffixed stems sharing the slug. A prefix glob alone proves + # nothing (a missing topic must never read, merge into, or delete a + # longer unrelated slug), so every candidate is validated against the + # record it actually holds. + prefix = f"{_slugify_topic(topic)}-" + matches = sorted(topics_dir.glob(f"{prefix}*.md")) + if not matches: + return None + for path in matches: + record = MemoryTopicRecord.from_markdown(path.read_text(encoding="utf-8"), fallback_topic=topic) + if record.topic == wanted: + return path + # A record carrying an explicit slug override stays findable by that + # slug; a default record never matches here, so colliding topics such + # as C++ and C# cannot merge through a shared slug. + for path in matches: + record = MemoryTopicRecord.from_markdown(path.read_text(encoding="utf-8"), fallback_topic=topic) + if record.slug == _slugify_topic(topic) and record.slug != _slugify_topic(record.topic): + return path + return None + @staticmethod def _serialize_json(value: object, *, dumps: JsonDumps) -> str: serialized = dumps(value) @@ -818,20 +909,29 @@ def list_topics(self, session: AgentSession, *, source_id: str) -> list[MemoryTo def get_topic(self, session: AgentSession, *, source_id: str, topic: str) -> MemoryTopicRecord: """Return one topic memory file by topic name or slug.""" - topic_path = self._topic_path(session, source_id=source_id, topic=topic) - if not topic_path.exists(): + topic_path = self._existing_topic_path(session, source_id=source_id, topic=topic) + if topic_path is None: raise FileNotFoundError(f"No memory topic named '{topic}' was found for this owner.") return MemoryTopicRecord.from_markdown(topic_path.read_text(encoding="utf-8"), fallback_topic=topic) def write_topic(self, session: AgentSession, record: MemoryTopicRecord, *, source_id: str) -> None: - """Persist one topic memory file.""" - topic_path = self._topic_path(session, source_id=source_id, topic=record.slug) + """Persist one topic memory file. + + The digest-stemmed file is the single authoritative record: when this + topic previously lived at another path (a legacy slug-only file, or a + stem written under a different slug override), that file is retired so + lookups can never surface stale content or resurrect the old record. + """ + previous_path = self._existing_topic_path(session, source_id=source_id, topic=record.topic) + topic_path = self._topic_path(session, source_id=source_id, topic=record.topic, slug_override=record.slug) _atomic_write_text(topic_path, f"{record.to_markdown()}\n") + if previous_path is not None and previous_path != topic_path: + previous_path.unlink(missing_ok=True) def delete_topic(self, session: AgentSession, *, source_id: str, topic: str) -> None: """Delete one topic memory file.""" - topic_path = self._topic_path(session, source_id=source_id, topic=topic) - if not topic_path.exists(): + topic_path = self._existing_topic_path(session, source_id=source_id, topic=topic) + if topic_path is None: raise FileNotFoundError(f"No memory topic named '{topic}' was found for this owner.") topic_path.unlink() diff --git a/python/packages/core/tests/core/test_harness_memory.py b/python/packages/core/tests/core/test_harness_memory.py index 464255d0b43..4e034be2de3 100644 --- a/python/packages/core/tests/core/test_harness_memory.py +++ b/python/packages/core/tests/core/test_harness_memory.py @@ -1046,6 +1046,160 @@ def test_memory_file_store_encodes_non_ascii_owner_ids(tmp_path) -> None: assert roots[0] != roots[1] +def test_memory_file_store_topics_with_the_same_slug_do_not_collide(tmp_path) -> None: + """Distinct topics that share a readable slug must keep separate memory files. + + ``C ++`` and ``C #`` both slugify to ``c``, so a slug-only filename let the + second topic's write destroy the first topic's memories. + """ + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + + first = MemoryTopicRecord( + topic="C++", summary="cpp", memories=["smart pointers"], updated_at="2026-10-26T00:00:00+00:00" + ) + second = MemoryTopicRecord( + topic="C#", summary="csharp", memories=["primary constructors"], updated_at="2026-10-26T00:00:00+00:00" + ) + store.write_topic(session, first, source_id=DEFAULT_MEMORY_SOURCE_ID) + store.write_topic(session, second, source_id=DEFAULT_MEMORY_SOURCE_ID) + + assert store.get_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="C++").memories == ["smart pointers"] + assert store.get_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="C#").memories == ["primary constructors"] + listed = sorted(entry.topic for entry in store.list_topics(session, source_id=DEFAULT_MEMORY_SOURCE_ID)) + assert listed == ["C#", "C++"] + + +def test_memory_file_store_reads_topics_from_the_legacy_slug_filename(tmp_path) -> None: + """Topic files written before the digest suffix stay readable.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + + topics_dir = store._get_topics_directory(session, source_id=DEFAULT_MEMORY_SOURCE_ID) + topics_dir.mkdir(parents=True, exist_ok=True) + (topics_dir / "python.md").write_text("# python\n\n## Memories\n\n- old note\n", encoding="utf-8") + + record = store.get_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="Python") + assert record.memories == ["old note"] + + +def test_memory_lookup_never_bleeds_across_slug_prefixes(tmp_path) -> None: + """A missing short topic must not read, merge into, or delete a longer + topic whose slug starts with the same characters.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + + store.write_topic( + session, + MemoryTopicRecord(topic="Collection", slug=None, summary="a", memories=["x"], updated_at="t"), + source_id="src", + ) + + with pytest.raises(FileNotFoundError): + store.get_topic(session, source_id="src", topic="coll") + + +def test_memory_lookup_validates_identity_for_colliding_slugs(tmp_path) -> None: + """Colliding topics such as C++ and C# resolve to their own records, and a + lookup for a topic that does not exist under a shared slug finds nothing + rather than an arbitrary same-slug file.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + + store.write_topic( + session, + MemoryTopicRecord(topic="C++", slug=None, summary="cpp", memories=["smart pointers"], updated_at="t"), + source_id="src", + ) + store.write_topic( + session, + MemoryTopicRecord(topic="C#", slug=None, summary="cs", memories=["ctors"], updated_at="t"), + source_id="src", + ) + + assert store.get_topic(session, source_id="src", topic="C++").memories == ["smart pointers"] + assert store.get_topic(session, source_id="src", topic="C#").memories == ["ctors"] + with pytest.raises(FileNotFoundError): + store.get_topic(session, source_id="src", topic="C") + + +def test_memory_write_retires_a_legacy_slug_file_for_the_same_topic(tmp_path) -> None: + """Rewriting a topic that lived at a legacy slug-only filename leaves a + single authoritative record: the digest-stemmed file.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + topics_dir = store._get_topics_directory(session, source_id="src") + + legacy = topics_dir / "legacy-topic.md" + legacy.parent.mkdir(parents=True, exist_ok=True) + legacy.write_text( + "# Legacy Topic\n\nUpdated: t0\nSessions: -\n\n## Summary\nold\n\n## Memories\n- old fact\n", + encoding="utf-8", + ) + + store.write_topic( + session, + MemoryTopicRecord(topic="Legacy Topic", slug=None, summary="new", memories=["new fact"], updated_at="t1"), + source_id="src", + ) + + assert not legacy.exists() + record = store.get_topic(session, source_id="src", topic="Legacy Topic") + assert record.memories == ["new fact"] + assert sorted(path.name for path in topics_dir.glob("legacy*.md")) == [f"{record.storage_stem}.md"] + + +def test_memory_index_pointer_links_resolve_to_the_persisted_files(tmp_path) -> None: + """Index entries link the persisted stem, so every pointer line names a + file that exists even when several topics share a readable slug.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + store.write_topic( + session, + MemoryTopicRecord(topic="C++", slug=None, summary="cpp", memories=["m"], updated_at="t"), + source_id="src", + ) + store.write_topic( + session, MemoryTopicRecord(topic="C#", slug=None, summary="cs", memories=["m"], updated_at="t"), source_id="src" + ) + topics_dir = store._get_topics_directory(session, source_id="src") + + entries = store.rebuild_index(session, source_id="src", line_limit=50, line_length=200) + + assert len(entries) == 2 + assert len({entry.slug for entry in entries}) == 2 + for entry in entries: + assert f"(topics/{entry.slug}.md)" in entry.to_pointer_line() + assert (topics_dir / f"{entry.slug}.md").exists() + + +def test_memory_explicit_slug_override_is_persisted_and_findable(tmp_path) -> None: + """A record with an explicit slug override persists under that slug and is + findable by it after a reload, without letting lookups for other names + bleed into the overridden record.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + + store.write_topic( + session, + MemoryTopicRecord(topic="Machine Learning", slug="ml-notes", summary="s", memories=["fact"], updated_at="t"), + source_id="src", + ) + + record = store.get_topic(session, source_id="src", topic="ml-notes") + assert record.topic == "Machine Learning" + assert record.memories == ["fact"] + with pytest.raises(FileNotFoundError): + store.get_topic(session, source_id="src", topic="machine") + + def test_memory_file_store_uses_literal_folders_for_safe_identifiers(tmp_path) -> None: """Safe identifiers are readable on disk rather than opaque encoded segments.""" store = MemoryFileStore(tmp_path, owner_prefix="user_", owner_state_key="owner_id") From cef2b8e95aacc8be4e87e8574c279116891da8ea Mon Sep 17 00:00:00 2001 From: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:01:10 +0530 Subject: [PATCH 2/8] Python: make memory topic stems collision safe and reloadable by slug The topic digest widens to 64 bits and write_topic verifies the occupant of an existing stem before reusing it, extending the digest when another topic already lives there. Slug lookups answer a single unambiguous holder again so default slug reloads work, and consolidation reloads each topic by exact identity instead of by slug. Signed-off-by: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> --- .../core/agent_framework/_harness/_memory.py | 157 ++++++++++++------ .../core/tests/core/test_harness_memory.py | 112 +++++++++++++ 2 files changed, 214 insertions(+), 55 deletions(-) diff --git a/python/packages/core/agent_framework/_harness/_memory.py b/python/packages/core/agent_framework/_harness/_memory.py index ea6ad95f244..0d8c99c26ac 100644 --- a/python/packages/core/agent_framework/_harness/_memory.py +++ b/python/packages/core/agent_framework/_harness/_memory.py @@ -12,7 +12,7 @@ from abc import ABC, abstractmethod from base64 import b32decode from binascii import Error as BinasciiError -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, ClassVar, Final, cast @@ -131,19 +131,31 @@ def _slugify_topic(topic: str) -> str: def _topic_stem(topic: str, *, slug_override: str | None = None) -> str: - """Return the injective filename stem for a topic. + """Return the primary filename stem for a topic. Distinct topics can share a readable slug: ``C ++`` and ``C #`` both slugify to ``c``, so a slug-only stem let the second topic silently overwrite the first topic's memories. The stem keeps the readable slug for humans and - appends a short digest of the normalized topic, so distinct normalized - topics never share a file (mirroring how ``_storage_key_segment`` keeps + appends a digest of the normalized topic, so distinct normalized topics + never share a file (mirroring how ``_storage_key_segment`` keeps byte-distinct identifiers in separate storage locations). """ + return next(_topic_stem_candidates(topic, slug_override=slug_override)) + + +def _topic_stem_candidates(topic: str, *, slug_override: str | None = None) -> Iterator[str]: + """Yield the stems a topic may persist under, longest digest last. + + The 64-bit digest makes a collision between distinct topics unrealistic, + and ``MemoryFileStore._writable_topic_path`` walks these candidates and + verifies the occupant of an existing stem before reusing it, so even a + real digest collision cannot make one topic overwrite another. + """ normalized = _normalize_topic(topic) slug = _slugify_topic(slug_override if slug_override is not None else normalized) - digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:8] - return f"{slug}-{digest}" + digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest() + for digest_length in (16, 32, 64): + yield f"{slug}-{digest[:digest_length]}" def _timestamp(value: datetime | None = None) -> str: @@ -362,7 +374,7 @@ class MemoryTopicRecord: memories: list[str] updated_at: str session_ids: list[str] - __slots__ = ("memories", "session_ids", "slug", "summary", "topic", "updated_at") + __slots__ = ("_on_disk_stem", "memories", "session_ids", "slug", "summary", "topic", "updated_at") def __init__( self, @@ -391,11 +403,18 @@ def __init__( self.summary = _coerce_summary(summary, self.memories) self.updated_at = updated_at self.session_ids = _dedupe_strings(session_ids or []) + self._on_disk_stem: str | None = None @property def storage_stem(self) -> str: - """Filename stem this record persists under (slug or explicit override plus digest).""" - return _topic_stem(self.topic, slug_override=self.slug) + """Filename stem this record persists under (slug or explicit override plus digest). + + Set from the file the record was read from or written to, so it stays + truthful even when a digest collision pushed the record onto an + extended stem; records that have not touched disk yet fall back to the + computed stem. + """ + return self._on_disk_stem or _topic_stem(self.topic, slug_override=self.slug) def to_dict(self) -> dict[str, Any]: """Serialize the topic record for tool output. @@ -796,29 +815,21 @@ def _get_index_path(self, session: AgentSession, *, source_id: str) -> Path: def _get_state_path(self, session: AgentSession, *, source_id: str) -> Path: return self._get_memory_root(session, source_id=source_id) / self.state_file_name - def _topic_path( - self, session: AgentSession, *, source_id: str, topic: str, slug_override: str | None = None - ) -> Path: - return ( - self._get_topics_directory(session, source_id=source_id) - / f"{_topic_stem(topic, slug_override=slug_override)}.md" - ) - - def _legacy_topic_path(self, session: AgentSession, *, source_id: str, topic: str) -> Path: - """Filename stem used before the digest suffix (kept for reading old stores).""" - return self._get_topics_directory(session, source_id=source_id) / f"{_slugify_topic(topic)}.md" - def _existing_topic_path(self, session: AgentSession, *, source_id: str, topic: str) -> Path | None: """Return the topic file to read, or ``None`` when no file matches. - Lookups resolve in order: the digest stem of the given name (the exact - file ``write_topic`` uses for that topic), a legacy slug-only file from - stores written before the digest suffix, then any file whose stem starts - with the readable slug followed by a dash, which serves lookups made by - slug or by a topic that normalizes differently than its stored spelling. - When several topics share a slug, a slug-only lookup is ambiguous by - nature; the file whose recorded topic matches the lookup wins, else the - first stem in sorted order keeps the choice deterministic. + Lookups resolve in order: the digest-stemmed candidates for the given + name (the files ``write_topic`` uses for that topic, including a stem + extended past a digest collision), an exact filename match (a legacy + slug-only file from stores written before the digest suffix, or a + persisted stem), then any file whose stem starts with the readable + slug followed by a dash, which serves lookups made by slug or by a + topic that normalizes differently than its stored spelling. Every + digest-stemmed or slug-family candidate is validated against the + record it actually holds, and a slug lookup answers only when the + lookup string is itself a slug with a single claiming record: + default-slug reloads stay working, while colliding topics such as + C++ and C# never merge through the shared slug. """ topics_dir = self._get_topics_directory(session, source_id=source_id) if not topics_dir.is_dir(): @@ -827,34 +838,40 @@ def _existing_topic_path(self, session: AgentSession, *, source_id: str, topic: wanted = _normalize_topic(topic) except ValueError: return None - # 1. The digest stem of the given name: the exact file write_topic uses. - topic_path = self._topic_path(session, source_id=source_id, topic=topic) - if topic_path.exists(): - return topic_path - # 2. An exact filename match: a legacy slug-only file from stores written - # before the digest suffix, or a caller passing a persisted stem. + # 1. The digest-stemmed candidates for this name: the exact files + # write_topic uses, each validated against the record it holds so a + # digest collision with another topic can never answer the lookup. + for candidate in _topic_stem_candidates(topic): + candidate_path = topics_dir / f"{candidate}.md" + if not candidate_path.exists(): + continue + record = MemoryTopicRecord.from_markdown(candidate_path.read_text(encoding="utf-8"), fallback_topic=topic) + if record.topic == wanted: + return candidate_path + # 2. An exact filename match: a legacy slug-only file from stores + # written before the digest suffix, or a caller passing a persisted + # stem such as an index entry slug. exact_path = topics_dir / f"{_slugify_topic(topic)}.md" if exact_path.exists(): return exact_path # 3. Digest-suffixed stems sharing the slug. A prefix glob alone proves # nothing (a missing topic must never read, merge into, or delete a - # longer unrelated slug), so every candidate is validated against the - # record it actually holds. - prefix = f"{_slugify_topic(topic)}-" - matches = sorted(topics_dir.glob(f"{prefix}*.md")) - if not matches: - return None - for path in matches: + # longer unrelated slug), so every candidate is validated against + # the record it actually holds. A slug lookup answers only when the + # lookup string is itself a slug and exactly one record claims it: + # default-slug reloads stay working, while a topic name whose + # slugification merely lands in the family (two CJK topics both + # slugifying to ``memory-topic``) cannot borrow a neighbor's file. + slug = _slugify_topic(topic) + slug_holders: list[Path] = [] + for path in sorted(topics_dir.glob(f"{slug}-*.md")): record = MemoryTopicRecord.from_markdown(path.read_text(encoding="utf-8"), fallback_topic=topic) if record.topic == wanted: return path - # A record carrying an explicit slug override stays findable by that - # slug; a default record never matches here, so colliding topics such - # as C++ and C# cannot merge through a shared slug. - for path in matches: - record = MemoryTopicRecord.from_markdown(path.read_text(encoding="utf-8"), fallback_topic=topic) - if record.slug == _slugify_topic(topic) and record.slug != _slugify_topic(record.topic): - return path + if record.slug == slug: + slug_holders.append(path) + if topic == slug and len(slug_holders) == 1: + return slug_holders[0] return None @staticmethod @@ -904,6 +921,7 @@ def list_topics(self, session: AgentSession, *, source_id: str) -> list[MemoryTo topic_path.read_text(encoding="utf-8"), fallback_topic=topic_path.stem.replace("-", " "), ) + record._on_disk_stem = topic_path.stem topics.append(record) return sorted(topics, key=lambda record: (record.topic.lower(), record.updated_at)) @@ -912,19 +930,45 @@ def get_topic(self, session: AgentSession, *, source_id: str, topic: str) -> Mem topic_path = self._existing_topic_path(session, source_id=source_id, topic=topic) if topic_path is None: raise FileNotFoundError(f"No memory topic named '{topic}' was found for this owner.") - return MemoryTopicRecord.from_markdown(topic_path.read_text(encoding="utf-8"), fallback_topic=topic) + record = MemoryTopicRecord.from_markdown(topic_path.read_text(encoding="utf-8"), fallback_topic=topic) + record._on_disk_stem = topic_path.stem + return record + + def _writable_topic_path(self, session: AgentSession, *, source_id: str, record: MemoryTopicRecord) -> Path: + """Return the path this record must persist under. + + A stem that is free, or already holds this topic, is reused. When + another topic lives at the primary stem (a digest collision between + distinct normalized topics), the digest extends until a free stem is + found, so a write can never silently replace a different topic's + memories. + """ + topics_dir = self._get_topics_directory(session, source_id=source_id) + normalized = _normalize_topic(record.topic) + for candidate in _topic_stem_candidates(record.topic, slug_override=record.slug): + candidate_path = topics_dir / f"{candidate}.md" + if not candidate_path.exists(): + return candidate_path + existing = MemoryTopicRecord.from_markdown( + candidate_path.read_text(encoding="utf-8"), fallback_topic=record.topic + ) + if _normalize_topic(existing.topic) == normalized: + return candidate_path + raise RuntimeError(f"Unable to allocate a distinct memory topic filename for '{record.topic}'.") def write_topic(self, session: AgentSession, record: MemoryTopicRecord, *, source_id: str) -> None: """Persist one topic memory file. The digest-stemmed file is the single authoritative record: when this - topic previously lived at another path (a legacy slug-only file, or a - stem written under a different slug override), that file is retired so - lookups can never surface stale content or resurrect the old record. + topic previously lived at another path (a legacy slug-only file, an + extended stem freed by a neighbor's deletion, or a stem written under a + different slug override), that file is retired so lookups can never + surface stale content or resurrect the old record. """ previous_path = self._existing_topic_path(session, source_id=source_id, topic=record.topic) - topic_path = self._topic_path(session, source_id=source_id, topic=record.topic, slug_override=record.slug) + topic_path = self._writable_topic_path(session, source_id=source_id, record=record) _atomic_write_text(topic_path, f"{record.to_markdown()}\n") + record._on_disk_stem = topic_path.stem if previous_path is not None and previous_path != topic_path: previous_path.unlink(missing_ok=True) @@ -1663,7 +1707,10 @@ async def _run_consolidation( for record in topic_records: async with self._topic_async_lock(session, record.slug): try: - current_record = self.store.get_topic(session, source_id=self.source_id, topic=record.slug) + # Reload by the exact topic identity: a slug can be shared + # by several topics, and dropping one of them here would + # silently skip its consolidation. + current_record = self.store.get_topic(session, source_id=self.source_id, topic=record.topic) except FileNotFoundError: continue consolidated_record, succeeded = await self._consolidate_topic( diff --git a/python/packages/core/tests/core/test_harness_memory.py b/python/packages/core/tests/core/test_harness_memory.py index 4e034be2de3..081dad57e80 100644 --- a/python/packages/core/tests/core/test_harness_memory.py +++ b/python/packages/core/tests/core/test_harness_memory.py @@ -28,6 +28,7 @@ Message, ) from agent_framework import _filesystem as filesystem_module +from agent_framework._harness._memory import _topic_stem from .test_filesystem import COLLIDING_IDENTIFIERS @@ -1200,6 +1201,117 @@ def test_memory_explicit_slug_override_is_persisted_and_findable(tmp_path) -> No store.get_topic(session, source_id="src", topic="machine") +def test_memory_topic_is_reloadable_by_its_default_slug(tmp_path) -> None: + """A normally written topic reloads by its default slug, which callers + holding record.slug rely on.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + + record = MemoryTopicRecord(topic="Travel Preferences", summary="s", memories=["window seat"], updated_at="t") + store.write_topic(session, record, source_id="src") + + reloaded = store.get_topic(session, source_id="src", topic=record.slug) + assert reloaded.topic == "Travel Preferences" + assert reloaded.memories == ["window seat"] + + +def test_memory_topics_colliding_on_the_digest_keep_separate_files(tmp_path) -> None: + """Two topics whose normalized names share a short digest must keep separate files. + + 主题乚丢一 and 主题亊不七 both digest to 74043f1d in their first eight hex + characters and both slugify to memory-topic, so a shorter digest let the + second write silently replace the first topic's memories, and reading the + first topic returned the second topic's fact. + """ + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + + first_topic = "主题乚丢一" + second_topic = "主题亊不七" + store.write_topic( + session, + MemoryTopicRecord(topic=first_topic, summary="a", memories=["fact a"], updated_at="t"), + source_id="src", + ) + store.write_topic( + session, + MemoryTopicRecord(topic=second_topic, summary="b", memories=["fact b"], updated_at="t"), + source_id="src", + ) + + assert store.get_topic(session, source_id="src", topic=first_topic).memories == ["fact a"] + assert store.get_topic(session, source_id="src", topic=second_topic).memories == ["fact b"] + listed = {record.topic: record.memories for record in store.list_topics(session, source_id="src")} + assert listed == {first_topic: ["fact a"], second_topic: ["fact b"]} + + +def test_memory_write_extends_the_stem_when_another_topic_owns_the_primary_stem(tmp_path) -> None: + """A write must never replace a different topic that already lives at the + primary stem; the digest extends until the stem is free.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + topics_dir = store._get_topics_directory(session, source_id="src") + + alpha = MemoryTopicRecord(topic="Alpha Topic", summary="a", memories=["alpha fact"], updated_at="t") + beta = MemoryTopicRecord(topic="Beta Topic", summary="b", memories=["beta fact"], updated_at="t") + primary = topics_dir / f"{_topic_stem('Beta Topic')}.md" + primary.parent.mkdir(parents=True, exist_ok=True) + primary.write_text(f"{alpha.to_markdown()}\n", encoding="utf-8") + + store.write_topic(session, beta, source_id="src") + + reread_alpha = MemoryTopicRecord.from_markdown(primary.read_text(encoding="utf-8")) + assert reread_alpha.memories == ["alpha fact"] + beta_files = [path for path in sorted(topics_dir.glob("beta-topic-*.md")) if path != primary] + assert len(beta_files) == 1 + assert beta_files[0].stem != primary.stem + assert beta.storage_stem == beta_files[0].stem + assert store.get_topic(session, source_id="src", topic="Beta Topic").memories == ["beta fact"] + + entries = store.rebuild_index(session, source_id="src", line_limit=10, line_length=200) + for entry in entries: + assert (topics_dir / f"{entry.slug}.md").exists() + + +async def test_memory_consolidation_processes_topics_whose_slug_differs_from_their_name(tmp_path) -> None: + """Forced consolidation reloads each topic by exact identity, so a topic + whose default slug differs from its name is consolidated rather than + silently skipped.""" + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + consolidation_client = _MemoryHarnessClient( + consolidation_payload={"summary": "Consolidated summary.", "memories": ["Visit Oslo in June."]} + ) + provider = MemoryContextProvider( + store=store, + consolidation_client=consolidation_client, # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + ) + agent = Agent( + client=_MemoryHarnessClient(), # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + context_providers=[provider], + default_options=_no_store_options(), + ) + + _, options = await agent._prepare_session_and_messages( # type: ignore[reportPrivateUsage] + session=session, + input_messages=[Message(role="user", contents=["Remember this."])], + ) + write_memory = _tool_by_name(options["tools"], "write_memory") + consolidate_memories = _tool_by_name(options["tools"], "consolidate_memories") + + await write_memory.invoke(arguments={"topic": "Travel Preferences", "memory": "Window seat."}) # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + await consolidate_memories.invoke() # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + + topic = store.get_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="Travel Preferences") + assert topic.summary == "Consolidated summary." + assert topic.memories == ["Visit Oslo in June."] + assert consolidation_client.calls == ["consolidate"] + + def test_memory_file_store_uses_literal_folders_for_safe_identifiers(tmp_path) -> None: """Safe identifiers are readable on disk rather than opaque encoded segments.""" store = MemoryFileStore(tmp_path, owner_prefix="user_", owner_state_key="owner_id") From f1fd7fa1b0f1b48dcac441f4ab23c47454563cc9 Mon Sep 17 00:00:00 2001 From: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> Date: Sat, 31 Oct 2026 07:58:40 +0530 Subject: [PATCH 3/8] Python: stamp the memory record stem through a setter Package Checks flagged the store assigning the record's private on disk stem field from outside the class. The stamp now goes through a small setter so the assignment happens inside the record. Signed-off-by: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> --- .../packages/core/agent_framework/_harness/_memory.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/packages/core/agent_framework/_harness/_memory.py b/python/packages/core/agent_framework/_harness/_memory.py index 0d8c99c26ac..176c5ef39b5 100644 --- a/python/packages/core/agent_framework/_harness/_memory.py +++ b/python/packages/core/agent_framework/_harness/_memory.py @@ -416,6 +416,10 @@ def storage_stem(self) -> str: """ return self._on_disk_stem or _topic_stem(self.topic, slug_override=self.slug) + def set_on_disk_stem(self, stem: str) -> None: + """Record the filename stem this record was read from or written to.""" + self._on_disk_stem = stem + def to_dict(self) -> dict[str, Any]: """Serialize the topic record for tool output. @@ -921,7 +925,7 @@ def list_topics(self, session: AgentSession, *, source_id: str) -> list[MemoryTo topic_path.read_text(encoding="utf-8"), fallback_topic=topic_path.stem.replace("-", " "), ) - record._on_disk_stem = topic_path.stem + record.set_on_disk_stem(topic_path.stem) topics.append(record) return sorted(topics, key=lambda record: (record.topic.lower(), record.updated_at)) @@ -931,7 +935,7 @@ def get_topic(self, session: AgentSession, *, source_id: str, topic: str) -> Mem if topic_path is None: raise FileNotFoundError(f"No memory topic named '{topic}' was found for this owner.") record = MemoryTopicRecord.from_markdown(topic_path.read_text(encoding="utf-8"), fallback_topic=topic) - record._on_disk_stem = topic_path.stem + record.set_on_disk_stem(topic_path.stem) return record def _writable_topic_path(self, session: AgentSession, *, source_id: str, record: MemoryTopicRecord) -> Path: @@ -968,7 +972,7 @@ def write_topic(self, session: AgentSession, record: MemoryTopicRecord, *, sourc previous_path = self._existing_topic_path(session, source_id=source_id, topic=record.topic) topic_path = self._writable_topic_path(session, source_id=source_id, record=record) _atomic_write_text(topic_path, f"{record.to_markdown()}\n") - record._on_disk_stem = topic_path.stem + record.set_on_disk_stem(topic_path.stem) if previous_path is not None and previous_path != topic_path: previous_path.unlink(missing_ok=True) From 0a7bbf53deccf282ed4c8b2d49f0ad92cce23135 Mon Sep 17 00:00:00 2001 From: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> Date: Tue, 3 Nov 2026 09:32:12 +0530 Subject: [PATCH 4/8] Python: bound memory topic stems to the filename component limit A slug long enough that its digest candidate plus the .md extension overshoots the filesystem component limit made the topic unwritable, with the errno raised before any valid persisted or legacy stem was reached. The slug part of generated candidates is capped so every candidate stays storable, oversized names are skipped rather than probed, and regressions cover a long legacy lookup and a bounded write with an index stem reload. Signed-off-by: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> --- .../core/agent_framework/_harness/_memory.py | 36 +++++++++++++--- .../core/tests/core/test_harness_memory.py | 43 +++++++++++++++++++ 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/python/packages/core/agent_framework/_harness/_memory.py b/python/packages/core/agent_framework/_harness/_memory.py index 176c5ef39b5..b00d3b7440c 100644 --- a/python/packages/core/agent_framework/_harness/_memory.py +++ b/python/packages/core/agent_framework/_harness/_memory.py @@ -143,16 +143,34 @@ def _topic_stem(topic: str, *, slug_override: str | None = None) -> str: return next(_topic_stem_candidates(topic, slug_override=slug_override)) +# Universal filename component budget: the longest candidate stem plus the +# .md extension and one separator must stay within the common 255 byte limit, +# so the readable slug is capped long before any probe touches the filesystem. +_MAX_TOPIC_SLUG_BYTES: Final[int] = 255 - len(".md") - 1 - 64 + + +def _bounded_slug_part(slug: str) -> str: + """Cap the readable slug so every candidate filename stays storable.""" + encoded = slug.encode("utf-8") + if len(encoded) <= _MAX_TOPIC_SLUG_BYTES: + return slug + return encoded[:_MAX_TOPIC_SLUG_BYTES].decode("utf-8", errors="ignore") + + def _topic_stem_candidates(topic: str, *, slug_override: str | None = None) -> Iterator[str]: """Yield the stems a topic may persist under, longest digest last. The 64-bit digest makes a collision between distinct topics unrealistic, and ``MemoryFileStore._writable_topic_path`` walks these candidates and verifies the occupant of an existing stem before reusing it, so even a - real digest collision cannot make one topic overwrite another. + real digest collision cannot make one topic overwrite another. The slug + part is capped so the longest candidate plus the .md extension stays + within the common filename component limit; probing an oversized name + would raise ENAMETOOLONG before a valid persisted or legacy stem is + reached, making long topics unreadable. """ normalized = _normalize_topic(topic) - slug = _slugify_topic(slug_override if slug_override is not None else normalized) + slug = _bounded_slug_part(_slugify_topic(slug_override if slug_override is not None else normalized)) digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest() for digest_length in (16, 32, 64): yield f"{slug}-{digest[:digest_length]}" @@ -847,6 +865,8 @@ def _existing_topic_path(self, session: AgentSession, *, source_id: str, topic: # digest collision with another topic can never answer the lookup. for candidate in _topic_stem_candidates(topic): candidate_path = topics_dir / f"{candidate}.md" + if len(candidate.encode("utf-8")) + len(".md") > 255: + continue if not candidate_path.exists(): continue record = MemoryTopicRecord.from_markdown(candidate_path.read_text(encoding="utf-8"), fallback_topic=topic) @@ -855,9 +875,13 @@ def _existing_topic_path(self, session: AgentSession, *, source_id: str, topic: # 2. An exact filename match: a legacy slug-only file from stores # written before the digest suffix, or a caller passing a persisted # stem such as an index entry slug. - exact_path = topics_dir / f"{_slugify_topic(topic)}.md" - if exact_path.exists(): - return exact_path + legacy_name = _slugify_topic(topic) + if len(legacy_name.encode("utf-8")) + len(".md") > 255: + pass # An overlong legacy name cannot exist on a limit bound filesystem. + else: + exact_path = topics_dir / f"{legacy_name}.md" + if exact_path.exists(): + return exact_path # 3. Digest-suffixed stems sharing the slug. A prefix glob alone proves # nothing (a missing topic must never read, merge into, or delete a # longer unrelated slug), so every candidate is validated against @@ -951,6 +975,8 @@ def _writable_topic_path(self, session: AgentSession, *, source_id: str, record: normalized = _normalize_topic(record.topic) for candidate in _topic_stem_candidates(record.topic, slug_override=record.slug): candidate_path = topics_dir / f"{candidate}.md" + if len(candidate.encode("utf-8")) + len(".md") > 255: + continue if not candidate_path.exists(): return candidate_path existing = MemoryTopicRecord.from_markdown( diff --git a/python/packages/core/tests/core/test_harness_memory.py b/python/packages/core/tests/core/test_harness_memory.py index 081dad57e80..c062b68afb7 100644 --- a/python/packages/core/tests/core/test_harness_memory.py +++ b/python/packages/core/tests/core/test_harness_memory.py @@ -1201,6 +1201,49 @@ def test_memory_explicit_slug_override_is_persisted_and_findable(tmp_path) -> No store.get_topic(session, source_id="src", topic="machine") +def test_memory_reads_a_legacy_topic_file_with_a_long_slug(tmp_path) -> None: + """A legacy slug only filename near the component limit stays readable. + + Lookup probes digest candidates first; without bounding, the longest + candidate for a long slug overshoots the filesystem component limit before + the legacy name is reached. + """ + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + topics_dir = store._get_topics_directory(session, source_id="src") + + legacy_slug = "a" * 240 + topics_dir.mkdir(parents=True, exist_ok=True) + (topics_dir / f"{legacy_slug}.md").write_text( + f"# {legacy_slug}\n\nUpdated: t\nSessions: -\n\n## Summary\ns\n\n## Memories\n- legacy fact\n", + encoding="utf-8", + ) + + record = store.get_topic(session, source_id="src", topic=legacy_slug) + assert record.memories == ["legacy fact"] + + +def test_memory_long_topic_writes_and_reloads_within_the_filename_limit(tmp_path) -> None: + """A topic whose slug overshoots the component limit still writes and reloads.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + + long_topic = " ".join(["engineering"] * 48) + written = MemoryTopicRecord(topic=long_topic, summary="s", memories=["kept"], updated_at="t") + store.write_topic(session, written, source_id="src") + + assert len(written.storage_stem.encode("utf-8")) + len(".md") <= 255 + assert store.get_topic(session, source_id="src", topic=long_topic).memories == ["kept"] + # Index entries link the same bounded stem and resolve after a reload. + by_stem = store.get_topic(session, source_id="src", topic=written.storage_stem) + assert by_stem.memories == ["kept"] + entries = store.rebuild_index(session, source_id="src", line_limit=10, line_length=400) + for entry in entries: + assert (store._get_topics_directory(session, source_id="src") / f"{entry.slug}.md").exists() + + def test_memory_topic_is_reloadable_by_its_default_slug(tmp_path) -> None: """A normally written topic reloads by its default slug, which callers holding record.slug rely on.""" From b87b00cd1cb40054edfb9c749ac3404468095878 Mon Sep 17 00:00:00 2001 From: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:49:44 +0530 Subject: [PATCH 5/8] Python: give memory topic aliases one identity for locks, reloads, and cleanup Locks key on the persisted stem so writes through any alias serialize with consolidation, consolidation locks and reloads by the same stem, retirement and deletion verify the occupant topic before unlinking, the bounded slug stays canonical after a separator boundary, and index trimming never cuts the link destination. Regressions cover the paused consolidation append, the ownership guard, the truncation boundary, and long index links. Signed-off-by: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> --- .../core/agent_framework/_harness/_memory.py | 78 ++++++-- .../core/tests/core/test_harness_memory.py | 177 ++++++++++++++++++ 2 files changed, 244 insertions(+), 11 deletions(-) diff --git a/python/packages/core/agent_framework/_harness/_memory.py b/python/packages/core/agent_framework/_harness/_memory.py index b00d3b7440c..feacaa87b44 100644 --- a/python/packages/core/agent_framework/_harness/_memory.py +++ b/python/packages/core/agent_framework/_harness/_memory.py @@ -154,7 +154,11 @@ def _bounded_slug_part(slug: str) -> str: encoded = slug.encode("utf-8") if len(encoded) <= _MAX_TOPIC_SLUG_BYTES: return slug - return encoded[:_MAX_TOPIC_SLUG_BYTES].decode("utf-8", errors="ignore") + # A cut can land right after a separator; trailing separators would pair + # with the digest separator into a double dash that a later slugify of the + # stem collapses, so the bounded slug stays canonical by dropping them. + bounded = encoded[:_MAX_TOPIC_SLUG_BYTES].decode("utf-8", errors="ignore") + return bounded.rstrip("-") def _topic_stem_candidates(topic: str, *, slug_override: str | None = None) -> Iterator[str]: @@ -365,8 +369,25 @@ def to_pointer_line(self, *, max_length: int = DEFAULT_MEMORY_INDEX_LINE_LENGTH) Returns: One compact pointer line. """ - pointer_line = f"- [{self.topic}](topics/{self.slug}.md): {self.summary}" - return _trim_pointer_line(pointer_line, max_length=max_length) + # The link destination must survive trimming: a long digest stem can + # push the line past the budget, and cutting the destination would + # strip the .md suffix and the closing parenthesis, leaving an index + # link that opens nothing. Trim the summary first, then the visible + # label, never the destination itself. + link_tail = f"](topics/{self.slug}.md): " + whole_link = f"[{self.topic}{link_tail}" + if len("- ") + len(whole_link) + len(self.summary) <= max_length: + return f"- {whole_link}{self.summary}" + summary_budget = max_length - len("- ") - len(whole_link) - len("...") + if summary_budget >= 0: + return f"- {whole_link}{self.summary[:summary_budget].rstrip()}..." + label_budget = max_length - len("- [") - len(link_tail) - len("...") + if label_budget > 0: + label = self.topic[:label_budget].rstrip() + return f"- [{label}{link_tail}..." + # Even the bare link destination busts the budget: keep the full line + # rather than emit a link whose destination was cut. + return f"- [{self.topic}{link_tail}{self.summary}" def __eq__(self, other: object) -> bool: """Compare two index entries by value.""" @@ -999,16 +1020,34 @@ def write_topic(self, session: AgentSession, record: MemoryTopicRecord, *, sourc topic_path = self._writable_topic_path(session, source_id=source_id, record=record) _atomic_write_text(topic_path, f"{record.to_markdown()}\n") record.set_on_disk_stem(topic_path.stem) - if previous_path is not None and previous_path != topic_path: + if ( + previous_path is not None + and previous_path != topic_path + and self._path_holds_topic(previous_path, record.topic) + ): + # A resolution through a legacy slug-only file or a shared slug can + # name another topic's record; retiring it would silently destroy + # that topic, so only a file holding this very topic is retired. previous_path.unlink(missing_ok=True) def delete_topic(self, session: AgentSession, *, source_id: str, topic: str) -> None: """Delete one topic memory file.""" topic_path = self._existing_topic_path(session, source_id=source_id, topic=topic) - if topic_path is None: + if topic_path is None or not self._path_holds_topic(topic_path, topic): raise FileNotFoundError(f"No memory topic named '{topic}' was found for this owner.") topic_path.unlink() + def _path_holds_topic(self, path: Path, topic: str) -> bool: + """Return whether the record in ``path`` is the given topic's own record. + + Digest-stemmed resolutions already validated exact identity; this guards + the alias-capable resolutions (legacy slug-only files, shared slugs), + where the occupant may be a different topic. The comparison folds case + because a legacy heading may spell the topic differently than a lookup. + """ + record = MemoryTopicRecord.from_markdown(path.read_text(encoding="utf-8"), fallback_topic=path.stem) + return _normalize_topic(record.topic).casefold() == _normalize_topic(topic).casefold() + def rebuild_index( self, session: AgentSession, @@ -1253,7 +1292,21 @@ def _store_async_lock(self, session: AgentSession, *, kind: str, key: str) -> as return store_lock def _topic_async_lock(self, session: AgentSession, topic: str) -> asyncio.Lock: - return self._store_async_lock(session, kind="topic", key=_slugify_topic(topic)) + """Return the per topic lock, keyed on one canonical identity. + + Every supported alias of a topic (its name, its readable slug, or the + persisted stem an index link carries) must resolve to the same lock, or + a write through one alias can interleave with a consolidation holding + another alias's lock and be overwritten by the stale result. The + persisted stem is that identity: it names the one file that holds the + topic. A name with no file yet locks on its slugified name, which every + concurrent creator of that topic also computes. + """ + try: + record = self.store.get_topic(session, source_id=self.source_id, topic=topic) + except FileNotFoundError: + return self._store_async_lock(session, kind="topic", key=_slugify_topic(topic)) + return self._store_async_lock(session, kind="topic", key=record.storage_stem) def _state_async_lock(self, session: AgentSession) -> asyncio.Lock: return self._store_async_lock(session, kind="state", key="maintenance") @@ -1735,12 +1788,15 @@ async def _run_consolidation( success_count = 0 for record in topic_records: - async with self._topic_async_lock(session, record.slug): + async with self._topic_async_lock(session, record.storage_stem or record.slug): try: - # Reload by the exact topic identity: a slug can be shared - # by several topics, and dropping one of them here would - # silently skip its consolidation. - current_record = self.store.get_topic(session, source_id=self.source_id, topic=record.topic) + # Reload by the persisted stem: it is the one name that + # resolves this exact file for every record shape, + # including records stored under an explicit slug, whose + # topic-derived filename family would miss them. + current_record = self.store.get_topic( + session, source_id=self.source_id, topic=record.storage_stem or record.topic + ) except FileNotFoundError: continue consolidated_record, succeeded = await self._consolidate_topic( diff --git a/python/packages/core/tests/core/test_harness_memory.py b/python/packages/core/tests/core/test_harness_memory.py index c062b68afb7..53f0525b17e 100644 --- a/python/packages/core/tests/core/test_harness_memory.py +++ b/python/packages/core/tests/core/test_harness_memory.py @@ -1201,6 +1201,89 @@ def test_memory_explicit_slug_override_is_persisted_and_findable(tmp_path) -> No store.get_topic(session, source_id="src", topic="machine") +def test_memory_writing_a_shared_slug_name_keeps_the_other_topic(tmp_path) -> None: + """Writing a topic whose name shares another topic's slug must not retire it.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + + store.write_topic( + session, + MemoryTopicRecord(topic="C++", slug=None, summary="cpp", memories=["smart pointers"], updated_at="t"), + source_id="src", + ) + store.write_topic( + session, + MemoryTopicRecord(topic="c", slug=None, summary="cee", memories=["letter"], updated_at="t"), + source_id="src", + ) + + assert store.get_topic(session, source_id="src", topic="C++").memories == ["smart pointers"] + assert store.get_topic(session, source_id="src", topic="c").memories == ["letter"] + with pytest.raises(FileNotFoundError): + store.delete_topic(session, source_id="src", topic="c2") + store.delete_topic(session, source_id="src", topic="c") + assert store.get_topic(session, source_id="src", topic="C++").memories == ["smart pointers"] + + +def test_memory_writing_a_new_slug_keeps_an_unrelated_legacy_slug_file(tmp_path) -> None: + """A legacy slug-only file holding another topic is never retired by a write.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + topics_dir = store._get_topics_directory(session, source_id="src") + + legacy = topics_dir / "c.md" + legacy.parent.mkdir(parents=True, exist_ok=True) + legacy.write_text( + "# C++\n\nUpdated: t\nSessions: -\n\n## Summary\ncpp\n\n## Memories\n- smart pointers\n", + encoding="utf-8", + ) + + store.write_topic( + session, + MemoryTopicRecord(topic="C#", slug=None, summary="cs", memories=["ctors"], updated_at="t"), + source_id="src", + ) + + assert store.get_topic(session, source_id="src", topic="C++").memories == ["smart pointers"] + assert legacy.exists() + + +def test_memory_topic_truncation_after_a_separator_stays_canonical(tmp_path) -> None: + """A slug cut right after a separator must not produce a double dash stem.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + + topic = "a" * 186 + " " + "b" * 10 + written = MemoryTopicRecord(topic=topic, summary="s", memories=["kept"], updated_at="t") + store.write_topic(session, written, source_id="src") + + assert "--" not in written.storage_stem + assert store.get_topic(session, source_id="src", topic=topic).memories == ["kept"] + entries = store.rebuild_index(session, source_id="src", line_limit=10, line_length=200) + for entry in entries: + assert (store._get_topics_directory(session, source_id="src") / f"{entry.slug}.md").exists() + assert store.get_topic(session, source_id="src", topic=entry.slug).memories == ["kept"] + + +def test_memory_index_pointer_lines_keep_the_full_link_for_long_names(tmp_path) -> None: + """Trimming a long pointer line shortens the label and summary, never the link.""" + entry = MemoryIndexEntry( + topic="release " + "notes " * 18, + slug="release-notes-" + "f" * 16, + summary="quarterly planning review with the product and research teams across every region", + updated_at="t", + ) + line = entry.to_pointer_line(max_length=150) + assert len(line) <= 150 + assert line.endswith(f"](topics/{entry.slug}.md): ...") or line.endswith("...") + assert f"](topics/{entry.slug}.md)" in line + parsed_slug = line.split("(topics/", 1)[1].split(".md)", 1)[0] + assert parsed_slug == entry.slug + + def test_memory_reads_a_legacy_topic_file_with_a_long_slug(tmp_path) -> None: """A legacy slug only filename near the component limit stays readable. @@ -1319,6 +1402,100 @@ def test_memory_write_extends_the_stem_when_another_topic_owns_the_primary_stem( assert (topics_dir / f"{entry.slug}.md").exists() +async def test_memory_consolidation_processes_records_stored_under_an_explicit_slug(tmp_path) -> None: + """Consolidation reloads by the persisted stem, so override slug records are not skipped.""" + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + consolidation_client = _MemoryHarnessClient( + consolidation_payload={"summary": "Consolidated override.", "memories": ["Visit Oslo in June."]} + ) + provider = MemoryContextProvider( + store=store, + consolidation_client=consolidation_client, # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + ) + agent = Agent( + client=_MemoryHarnessClient(), # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + context_providers=[provider], + default_options=_no_store_options(), + ) + + _, options = await agent._prepare_session_and_messages( # type: ignore[reportPrivateUsage] + session=session, + input_messages=[Message(role="user", contents=["Remember this."])], + ) + consolidate_memories = _tool_by_name(options["tools"], "consolidate_memories") + + store.write_topic( + session, + MemoryTopicRecord( + topic="Machine Learning", + slug="ml-notes", + summary="s", + memories=["prefers short answers"], + updated_at="t", + ), + source_id=DEFAULT_MEMORY_SOURCE_ID, + ) + await consolidate_memories.invoke() # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + + record = store.get_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="ml-notes") + assert record.topic == "Machine Learning" + assert record.summary == "Consolidated override." + assert consolidation_client.calls == ["consolidate"] + + +async def test_memory_consolidation_does_not_lose_a_write_through_an_index_stem(tmp_path) -> None: + """A write through a persisted stem alias during consolidation must survive it.""" + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + provider = MemoryContextProvider(store=store) + agent = Agent( + client=_MemoryHarnessClient(), # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + context_providers=[provider], + default_options=_no_store_options(), + ) + + _, options = await agent._prepare_session_and_messages( # type: ignore[reportPrivateUsage] + session=session, + input_messages=[Message(role="user", contents=["Remember this."])], + ) + write_memory = _tool_by_name(options["tools"], "write_memory") + await write_memory.invoke(arguments={"topic": "Travel Preferences", "memory": "window seat"}) # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] + written = store.get_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="Travel Preferences") + stem = written.storage_stem + + original_consolidate = provider._consolidate_topic # type: ignore[reportPrivateUsage] + pending_merge: dict[str, object] = {} + + async def consolidate_with_a_write_in_flight(*, client, record, now): # type: ignore[reportPrivateUsage] + # A write arrives through the index stem alias while consolidation is + # mid call. With one canonical lock identity it waits for the lock and + # then applies on top of the consolidated record; with alias keyed + # locks it would complete now and be overwritten by the stale result. + import asyncio as _asyncio + + task = _asyncio.create_task( + provider._merge_memory(session=session, topic=stem, memory="urgent fact", now=now) # type: ignore[reportPrivateUsage] + ) + pending_merge["task"] = task + await _asyncio.sleep(0) + await _asyncio.sleep(0) + return record, True + + provider._consolidate_topic = consolidate_with_a_write_in_flight # type: ignore[reportPrivateUsage] + await provider._run_consolidation( # type: ignore[reportPrivateUsage] + client=_MemoryHarnessClient(), session=session, force=True, now=datetime.now(timezone.utc) + ) + await pending_merge["task"] # type: ignore[misc] + + final_record = store.get_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="Travel Preferences") + assert "urgent fact" in final_record.memories + assert "window seat" in final_record.memories + del original_consolidate + + async def test_memory_consolidation_processes_topics_whose_slug_differs_from_their_name(tmp_path) -> None: """Forced consolidation reloads each topic by exact identity, so a topic whose default slug differs from its name is consolidated rather than From cbdc13da503f92019308d004a1deae7c39278919 Mon Sep 17 00:00:00 2001 From: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> Date: Tue, 10 Nov 2026 08:25:41 +0530 Subject: [PATCH 6/8] Python: let deletion and consolidation use every documented topic alias Deletion rejected the readable slug and persisted stem that get_topic answers, and consolidation reloaded by the file-specific stem, which made conforming non-file stores skip every record. Deletion now accepts every documented alias through the same resolution get_topic uses, and consolidation reloads by the persisted stem first with the topic name and slug as documented fallbacks, so a custom store resolves by topic. The unused pointer trim helper that tripped the package checks is removed and the paused consolidation test is fully typed. Signed-off-by: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> --- .../core/agent_framework/_harness/_memory.py | 52 ++++--- .../core/tests/core/test_harness_memory.py | 139 ++++++++++++++++-- 2 files changed, 161 insertions(+), 30 deletions(-) diff --git a/python/packages/core/agent_framework/_harness/_memory.py b/python/packages/core/agent_framework/_harness/_memory.py index feacaa87b44..c32023542cc 100644 --- a/python/packages/core/agent_framework/_harness/_memory.py +++ b/python/packages/core/agent_framework/_harness/_memory.py @@ -200,14 +200,6 @@ def _dedupe_strings(values: Sequence[str]) -> list[str]: return deduped -def _trim_pointer_line(pointer_line: str, *, max_length: int) -> str: - if len(pointer_line) <= max_length: - return pointer_line - if max_length <= 3: - return pointer_line[:max_length] - return f"{pointer_line[: max_length - 3].rstrip()}..." - - def _extract_json_text(text: str) -> str: stripped = text.strip() if stripped.startswith("```") and stripped.endswith("```"): @@ -1031,9 +1023,17 @@ def write_topic(self, session: AgentSession, record: MemoryTopicRecord, *, sourc previous_path.unlink(missing_ok=True) def delete_topic(self, session: AgentSession, *, source_id: str, topic: str) -> None: - """Delete one topic memory file.""" + """Delete one topic memory file. + + Deletion accepts every alias get_topic answers: the topic name, the + readable slug, and the persisted stem an index link carries. The lookup + resolution itself rejects an ambiguous slug and validates digest stems, + so the file it names is the one record the lookup resolves to; the + topic ownership guard stays on the write path, where a resolution can + name another topic's file for retirement. + """ topic_path = self._existing_topic_path(session, source_id=source_id, topic=topic) - if topic_path is None or not self._path_holds_topic(topic_path, topic): + if topic_path is None: raise FileNotFoundError(f"No memory topic named '{topic}' was found for this owner.") topic_path.unlink() @@ -1764,6 +1764,27 @@ async def _merge_memory( self.store.write_topic(session, updated_record, source_id=self.source_id) return updated_record + async def _reload_topic_record( + self, session: AgentSession, *, listed: MemoryTopicRecord + ) -> MemoryTopicRecord | None: + """Reload one listed record through the aliases the store documents. + + The file store answers the persisted stem first: it names this exact + record for every shape, including explicit slug records whose topic + derived family would miss and records whose slug another topic shares. + The topic name and the readable slug follow, both part of the + MemoryStore get_topic contract, so a conforming custom store resolves + by topic even though its records have no persisted stem. A record no + alias can find is skipped, the previous behaviour for a record deleted + between listing and lock. + """ + for alias in (listed.storage_stem, listed.topic, listed.slug): + try: + return self.store.get_topic(session, source_id=self.source_id, topic=alias) + except FileNotFoundError: + continue + return None + async def _run_consolidation( self, *, @@ -1789,15 +1810,8 @@ async def _run_consolidation( success_count = 0 for record in topic_records: async with self._topic_async_lock(session, record.storage_stem or record.slug): - try: - # Reload by the persisted stem: it is the one name that - # resolves this exact file for every record shape, - # including records stored under an explicit slug, whose - # topic-derived filename family would miss them. - current_record = self.store.get_topic( - session, source_id=self.source_id, topic=record.storage_stem or record.topic - ) - except FileNotFoundError: + current_record = await self._reload_topic_record(session, listed=record) + if current_record is None: continue consolidated_record, succeeded = await self._consolidate_topic( client=client, record=current_record, now=now diff --git a/python/packages/core/tests/core/test_harness_memory.py b/python/packages/core/tests/core/test_harness_memory.py index 53f0525b17e..bb4089a9d27 100644 --- a/python/packages/core/tests/core/test_harness_memory.py +++ b/python/packages/core/tests/core/test_harness_memory.py @@ -26,6 +26,7 @@ MemoryStore, MemoryTopicRecord, Message, + SupportsChatGetResponse, ) from agent_framework import _filesystem as filesystem_module from agent_framework._harness._memory import _topic_stem @@ -1327,6 +1328,25 @@ def test_memory_long_topic_writes_and_reloads_within_the_filename_limit(tmp_path assert (store._get_topics_directory(session, source_id="src") / f"{entry.slug}.md").exists() +def test_memory_topic_is_deletable_by_every_lookup_alias(tmp_path) -> None: + """Deletion accepts the same aliases get_topic answers. + + The slug and the persisted stem an index link carries both resolve through + get_topic; rejecting them in delete_topic left callers able to read a + record by an alias but unable to remove it by the same alias. + """ + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + + record = MemoryTopicRecord(topic="Travel Preferences", summary="s", memories=["window seat"], updated_at="t") + for alias in (record.slug, record.storage_stem, "Travel Preferences"): + store.write_topic(session, record, source_id="src") + store.delete_topic(session, source_id="src", topic=alias) + with pytest.raises(FileNotFoundError): + store.get_topic(session, source_id="src", topic="Travel Preferences") + + def test_memory_topic_is_reloadable_by_its_default_slug(tmp_path) -> None: """A normally written topic reloads by its default slug, which callers holding record.slug rely on.""" @@ -1467,28 +1487,36 @@ async def test_memory_consolidation_does_not_lose_a_write_through_an_index_stem( stem = written.storage_stem original_consolidate = provider._consolidate_topic # type: ignore[reportPrivateUsage] - pending_merge: dict[str, object] = {} + pending_merge: asyncio.Task[None] | None = None - async def consolidate_with_a_write_in_flight(*, client, record, now): # type: ignore[reportPrivateUsage] + async def consolidate_with_a_write_in_flight( + *, + client: SupportsChatGetResponse[Any] | None, + record: MemoryTopicRecord, + now: datetime, + ) -> tuple[MemoryTopicRecord, bool]: # A write arrives through the index stem alias while consolidation is # mid call. With one canonical lock identity it waits for the lock and # then applies on top of the consolidated record; with alias keyed # locks it would complete now and be overwritten by the stale result. - import asyncio as _asyncio - - task = _asyncio.create_task( + del client + nonlocal pending_merge + pending_merge = asyncio.create_task( provider._merge_memory(session=session, topic=stem, memory="urgent fact", now=now) # type: ignore[reportPrivateUsage] ) - pending_merge["task"] = task - await _asyncio.sleep(0) - await _asyncio.sleep(0) + await asyncio.sleep(0) + await asyncio.sleep(0) return record, True - provider._consolidate_topic = consolidate_with_a_write_in_flight # type: ignore[reportPrivateUsage] + provider._consolidate_topic = consolidate_with_a_write_in_flight # type: ignore[method-assign,reportPrivateUsage] await provider._run_consolidation( # type: ignore[reportPrivateUsage] - client=_MemoryHarnessClient(), session=session, force=True, now=datetime.now(timezone.utc) + client=_MemoryHarnessClient(), # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + session=session, + force=True, + now=datetime.now(timezone.utc), ) - await pending_merge["task"] # type: ignore[misc] + assert pending_merge is not None + await pending_merge final_record = store.get_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="Travel Preferences") assert "urgent fact" in final_record.memories @@ -1496,6 +1524,95 @@ async def consolidate_with_a_write_in_flight(*, client, record, now): # type: i del original_consolidate +class _AliasOnlyDictStore(MemoryStore): + """Conforming non-file store resolving get_topic by topic name or slug.""" + + def __init__(self) -> None: + self.topics: dict[str, MemoryTopicRecord] = {} + + def list_topics(self, session: AgentSession, *, source_id: str) -> list[MemoryTopicRecord]: + return list(self.topics.values()) + + def get_topic(self, session: AgentSession, *, source_id: str, topic: str) -> MemoryTopicRecord: + for record in self.topics.values(): + if record.topic == topic or record.slug == topic: + return record + raise FileNotFoundError(topic) + + def write_topic(self, session: AgentSession, record: MemoryTopicRecord, *, source_id: str) -> None: + self.topics[record.topic] = record + + def delete_topic(self, session: AgentSession, *, source_id: str, topic: str) -> None: + self.topics.pop(topic, None) + + def rebuild_index( + self, session: AgentSession, *, source_id: str, line_limit: int, line_length: int + ) -> list[MemoryIndexEntry]: + return [] + + def get_index_text( + self, + session: AgentSession, + *, + source_id: str, + line_limit: int, + line_length: int, + index_entries: Sequence[MemoryIndexEntry] | None = None, + ) -> str: + return "" + + def get_transcripts_directory(self, session: AgentSession, *, source_id: str) -> Path: + return Path(".") + + def search_transcripts(self, session: AgentSession, *, query: str, source_id: str) -> list[dict[str, Any]]: + return [] + + def read_state(self, session: AgentSession, *, source_id: str) -> dict[str, Any]: + return {} + + def write_state(self, session: AgentSession, state: dict[str, Any], *, source_id: str) -> None: + pass + + +async def test_memory_consolidation_processes_topics_in_a_conforming_custom_store(tmp_path) -> None: + """Consolidation reloads through documented aliases, not file-specific stems. + + A store whose get_topic answers the topic name and slug is the documented + contract; reloading by the persisted storage stem skipped every record in + such stores, so consolidation silently did nothing. + """ + store = _AliasOnlyDictStore() + provider = MemoryContextProvider(store=store) + session = AgentSession(session_id="session-1") + store.write_topic( + session, + MemoryTopicRecord(topic="Notes", summary="s", memories=["m"], updated_at="t"), + source_id=DEFAULT_MEMORY_SOURCE_ID, + ) + + class _DictClient: + async def get_response(self, *args: Any, **kwargs: Any) -> ChatResponse[Any]: + return ChatResponse( + messages=[ + Message( + role="assistant", + contents=[json.dumps({"summary": "consolidated", "memories": ["m"]})], + ) + ] + ) + + count = await provider._run_consolidation( # pyright: ignore[reportPrivateUsage] + client=_DictClient(), + session=session, + force=True, + now=datetime.now(timezone.utc), + ) + + assert count == 1 + reloaded = store.get_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="Notes") + assert reloaded.summary == "consolidated" + + async def test_memory_consolidation_processes_topics_whose_slug_differs_from_their_name(tmp_path) -> None: """Forced consolidation reloads each topic by exact identity, so a topic whose default slug differs from its name is consolidated rather than From 35c8ddfbddf6028ebbb274b4e9229a51e916ca64 Mon Sep 17 00:00:00 2001 From: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:46:16 +0530 Subject: [PATCH 7/8] Python: keep case distinct topics and the index slug contract intact The case folded ownership check let the slug shaped topic api retire the API record through their shared resolution, and the index entry factory put the file store stem in the public slug field that custom stores resolve. Ownership now compares exact normalized topics, the index entry carries the readable slug with the persisted stem in its own link-only field, and injection resolves through the stem, slug, and topic ladder, so conforming custom stores keep working. Regressions cover both case orders and a custom store end to end through before_run. Signed-off-by: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> --- .../core/agent_framework/_harness/_memory.py | 56 ++++++++++---- .../core/tests/core/test_harness_memory.py | 76 ++++++++++++++++--- 2 files changed, 106 insertions(+), 26 deletions(-) diff --git a/python/packages/core/agent_framework/_harness/_memory.py b/python/packages/core/agent_framework/_harness/_memory.py index c32023542cc..1991875a3da 100644 --- a/python/packages/core/agent_framework/_harness/_memory.py +++ b/python/packages/core/agent_framework/_harness/_memory.py @@ -280,21 +280,27 @@ class MemoryIndexEntry: slug: str summary: str updated_at: str - __slots__ = ("slug", "summary", "topic", "updated_at") + storage_stem: str | None + __slots__ = ("slug", "storage_stem", "summary", "topic", "updated_at") - def __init__(self, topic: str, slug: str, summary: str, updated_at: str) -> None: + def __init__(self, topic: str, slug: str, summary: str, updated_at: str, storage_stem: str | None = None) -> None: """Initialize one memory-index entry. Args: topic: Human-readable topic name. - slug: Stable topic filename stem. + slug: Stable readable topic slug, kept in the documented lookup + contract (stores resolve the topic name or this slug). summary: Short summary used by the index line. updated_at: Last update timestamp for the topic. + storage_stem: Optional persisted filename stem used as the link + destination when the store's files carry digest suffixes; + entries without one link their slug, as before. """ self.topic = _normalize_topic(topic) self.slug = _slugify_topic(slug) self.summary = _coerce_summary(summary, []) self.updated_at = updated_at + self.storage_stem = _slugify_topic(storage_stem) if storage_stem is not None else None def to_dict(self) -> dict[str, str]: """Serialize the index entry for tool output. @@ -345,11 +351,13 @@ def from_topic_record(cls, record: MemoryTopicRecord) -> MemoryIndexEntry: """ return cls( topic=record.topic, - # The persisted stem, not the readable slug: distinct topics can - # share a slug, and an index link must point at the real file. - slug=record.storage_stem, + # The readable slug stays in the public field: stores resolve the + # topic name or that slug. The persisted stem, unique per record, + # rides in its own field and only names the link destination. + slug=record.slug, summary=_coerce_summary(record.summary, record.memories), updated_at=record.updated_at, + storage_stem=record.storage_stem, ) def to_pointer_line(self, *, max_length: int = DEFAULT_MEMORY_INDEX_LINE_LENGTH) -> str: @@ -365,8 +373,10 @@ def to_pointer_line(self, *, max_length: int = DEFAULT_MEMORY_INDEX_LINE_LENGTH) # push the line past the budget, and cutting the destination would # strip the .md suffix and the closing parenthesis, leaving an index # link that opens nothing. Trim the summary first, then the visible - # label, never the destination itself. - link_tail = f"](topics/{self.slug}.md): " + # label, never the destination itself. The destination is the persisted + # stem when one rides along, else the readable slug, as before. + destination = self.storage_stem if self.storage_stem is not None else self.slug + link_tail = f"](topics/{destination}.md): " whole_link = f"[{self.topic}{link_tail}" if len("- ") + len(whole_link) + len(self.summary) <= max_length: return f"- {whole_link}{self.summary}" @@ -1042,11 +1052,12 @@ def _path_holds_topic(self, path: Path, topic: str) -> bool: Digest-stemmed resolutions already validated exact identity; this guards the alias-capable resolutions (legacy slug-only files, shared slugs), - where the occupant may be a different topic. The comparison folds case - because a legacy heading may spell the topic differently than a lookup. + where the occupant may be a different topic. The comparison is exact on + the normalized topic: case folding would treat slug shaped neighbours + such as API and api as one topic and retire the original file. """ record = MemoryTopicRecord.from_markdown(path.read_text(encoding="utf-8"), fallback_topic=path.stem) - return _normalize_topic(record.topic).casefold() == _normalize_topic(topic).casefold() + return _normalize_topic(record.topic) == _normalize_topic(topic) def rebuild_index( self, @@ -1458,9 +1469,24 @@ async def before_run( load_tool_turns=self.load_tool_turns, ) selected_entries = self._select_topics(entries=index_entries, input_messages=context.input_messages) - selected_topics = [ - self.store.get_topic(session, source_id=self.source_id, topic=entry.slug) for entry in selected_entries - ] + selected_topics: list[MemoryTopicRecord] = [] + for entry in selected_entries: + # The same alias ladder the reload uses: the persisted stem (the + # exact file for digest stemmed records), then the readable slug + # and the topic name, both part of the documented get_topic + # contract that custom stores resolve. + record: MemoryTopicRecord | None = None + for alias in (entry.storage_stem, entry.slug, entry.topic): + if alias is None: + continue + try: + record = self.store.get_topic(session, source_id=self.source_id, topic=alias) + break + except FileNotFoundError: + continue + if record is None: + raise FileNotFoundError(f"No memory topic named '{entry.slug}' was found for this owner.") + selected_topics.append(record) @tool(name="list_memory_topics", approval_mode="never_require") def list_memory_topics() -> str: @@ -1536,7 +1562,7 @@ async def consolidate_memories() -> str: ) return json.dumps({"consolidated_topics": consolidated_count}, ensure_ascii=False) - loaded_topic_blocks = [ + loaded_topic_blocks: list[str] = [ f"### topics/{record.slug}.md\n{record.to_markdown()}" for record in selected_topics ] or ["- none auto-loaded for this turn"] loaded_topics_text = "\n\n".join(loaded_topic_blocks) diff --git a/python/packages/core/tests/core/test_harness_memory.py b/python/packages/core/tests/core/test_harness_memory.py index bb4089a9d27..09776aa4dea 100644 --- a/python/packages/core/tests/core/test_harness_memory.py +++ b/python/packages/core/tests/core/test_harness_memory.py @@ -1175,10 +1175,10 @@ def test_memory_index_pointer_links_resolve_to_the_persisted_files(tmp_path) -> entries = store.rebuild_index(session, source_id="src", line_limit=50, line_length=200) assert len(entries) == 2 - assert len({entry.slug for entry in entries}) == 2 + assert len({entry.storage_stem for entry in entries}) == 2 for entry in entries: - assert f"(topics/{entry.slug}.md)" in entry.to_pointer_line() - assert (topics_dir / f"{entry.slug}.md").exists() + assert f"(topics/{entry.storage_stem}.md)" in entry.to_pointer_line() + assert (topics_dir / f"{entry.storage_stem}.md").exists() def test_memory_explicit_slug_override_is_persisted_and_findable(tmp_path) -> None: @@ -1265,8 +1265,8 @@ def test_memory_topic_truncation_after_a_separator_stays_canonical(tmp_path) -> assert store.get_topic(session, source_id="src", topic=topic).memories == ["kept"] entries = store.rebuild_index(session, source_id="src", line_limit=10, line_length=200) for entry in entries: - assert (store._get_topics_directory(session, source_id="src") / f"{entry.slug}.md").exists() - assert store.get_topic(session, source_id="src", topic=entry.slug).memories == ["kept"] + assert (store._get_topics_directory(session, source_id="src") / f"{entry.storage_stem}.md").exists() + assert store.get_topic(session, source_id="src", topic=entry.storage_stem).memories == ["kept"] def test_memory_index_pointer_lines_keep_the_full_link_for_long_names(tmp_path) -> None: @@ -1277,12 +1277,13 @@ def test_memory_index_pointer_lines_keep_the_full_link_for_long_names(tmp_path) summary="quarterly planning review with the product and research teams across every region", updated_at="t", ) + destination = entry.slug line = entry.to_pointer_line(max_length=150) assert len(line) <= 150 - assert line.endswith(f"](topics/{entry.slug}.md): ...") or line.endswith("...") - assert f"](topics/{entry.slug}.md)" in line + assert line.endswith(f"](topics/{destination}.md): ...") or line.endswith("...") + assert f"](topics/{destination}.md)" in line parsed_slug = line.split("(topics/", 1)[1].split(".md)", 1)[0] - assert parsed_slug == entry.slug + assert parsed_slug == destination def test_memory_reads_a_legacy_topic_file_with_a_long_slug(tmp_path) -> None: @@ -1325,7 +1326,29 @@ def test_memory_long_topic_writes_and_reloads_within_the_filename_limit(tmp_path assert by_stem.memories == ["kept"] entries = store.rebuild_index(session, source_id="src", line_limit=10, line_length=400) for entry in entries: - assert (store._get_topics_directory(session, source_id="src") / f"{entry.slug}.md").exists() + assert (store._get_topics_directory(session, source_id="src") / f"{entry.storage_stem}.md").exists() + + +def test_memory_topics_that_differ_only_by_case_stay_distinct(tmp_path) -> None: + """Slug shaped neighbours such as API and api must not merge through the + case folded ownership check, in either write order.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + + for first, second in (("API", "api"), ("api", "API")): + for name in (first, second): + store.write_topic( + session, + MemoryTopicRecord(topic=name, summary="s", memories=[name + " fact"], updated_at="t"), + source_id="src", + ) + assert store.get_topic(session, source_id="src", topic=first).memories == [first + " fact"] + assert store.get_topic(session, source_id="src", topic=second).memories == [second + " fact"] + topics_dir = store._get_topics_directory(session, source_id="src") + assert len(list(topics_dir.glob("*.md"))) == 2 + for path in topics_dir.glob("*.md"): + path.unlink() def test_memory_topic_is_deletable_by_every_lookup_alias(tmp_path) -> None: @@ -1419,7 +1442,7 @@ def test_memory_write_extends_the_stem_when_another_topic_owns_the_primary_stem( entries = store.rebuild_index(session, source_id="src", line_limit=10, line_length=200) for entry in entries: - assert (topics_dir / f"{entry.slug}.md").exists() + assert (topics_dir / f"{entry.storage_stem}.md").exists() async def test_memory_consolidation_processes_records_stored_under_an_explicit_slug(tmp_path) -> None: @@ -1548,7 +1571,7 @@ def delete_topic(self, session: AgentSession, *, source_id: str, topic: str) -> def rebuild_index( self, session: AgentSession, *, source_id: str, line_limit: int, line_length: int ) -> list[MemoryIndexEntry]: - return [] + return [MemoryIndexEntry.from_topic_record(r) for r in self.topics.values()][:line_limit] def get_index_text( self, @@ -1574,6 +1597,37 @@ def write_state(self, session: AgentSession, state: dict[str, Any], *, source_id pass +async def test_memory_before_run_injects_topics_from_a_conforming_custom_store() -> None: + """A custom store's entries resolve through the documented topic or slug. + + The index entry factory derives from topic records, so its output must stay + inside the documented get_topic contract: injection resolves the readable + slug and topic name for stores that have no persisted filename stem. + """ + store = _AliasOnlyDictStore() + provider = MemoryContextProvider(store=store) + session = AgentSession(session_id="session-1") + record = MemoryTopicRecord(topic="Notes", summary="s", memories=["kept"], updated_at="t") + store.write_topic(session, record, source_id=DEFAULT_MEMORY_SOURCE_ID) + + # The entry the file store style factory produces: readable slug, stem optional. + entry = MemoryIndexEntry.from_topic_record(record) + assert entry.slug == record.slug + + from agent_framework._sessions import SessionContext + + context = SessionContext( + input_messages=[Message(role="user", contents=["Read the notes topic."])], + ) + state: dict[str, Any] = {} + await provider.before_run(agent=None, session=session, context=context, state=state) # pyright: ignore[reportPrivateUsage,reportArgumentType] + + # before_run succeeded: the selected topic resolved through the documented + # slug or topic alias, and its content reached the injected messages. + injected_texts = [m.text for group in context.context_messages.values() for m in group] + assert any("kept" in text for text in injected_texts) + + async def test_memory_consolidation_processes_topics_in_a_conforming_custom_store(tmp_path) -> None: """Consolidation reloads through documented aliases, not file-specific stems. From 58a7ae00ed73841957c430689361910de0361b00 Mon Sep 17 00:00:00 2001 From: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:43:55 +0530 Subject: [PATCH 8/8] Python: validate legacy topic reads and reconcile interrupted migrations --- .../core/agent_framework/_harness/_memory.py | 201 +++++++++++++----- .../core/tests/core/test_harness_memory.py | 166 +++++++++++++-- 2 files changed, 299 insertions(+), 68 deletions(-) diff --git a/python/packages/core/agent_framework/_harness/_memory.py b/python/packages/core/agent_framework/_harness/_memory.py index 1991875a3da..190e5adf5dd 100644 --- a/python/packages/core/agent_framework/_harness/_memory.py +++ b/python/packages/core/agent_framework/_harness/_memory.py @@ -180,6 +180,20 @@ def _topic_stem_candidates(topic: str, *, slug_override: str | None = None) -> I yield f"{slug}-{digest[:digest_length]}" +_DIGEST_STEM_RE = re.compile(r"-(?:[0-9a-f]{16}|[0-9a-f]{32}|[0-9a-f]{64})$") + + +def _is_digest_stem(stem: str) -> bool: + """Return whether a filename stem carries the digest suffix ``write_topic`` appends. + + Digest-bearing stems are injective identities: ``write_topic`` extends the + digest when a different topic already lives at the stem, so one stem names + one topic's file. A slug-only stem is not injective, which is why lookups + validate the record such a file holds instead of trusting the name. + """ + return _DIGEST_STEM_RE.search(stem) is not None + + def _timestamp(value: datetime | None = None) -> str: current = value or datetime.now(timezone.utc) return current.replace(microsecond=0).isoformat() @@ -870,11 +884,16 @@ def _existing_topic_path(self, session: AgentSession, *, source_id: str, topic: persisted stem), then any file whose stem starts with the readable slug followed by a dash, which serves lookups made by slug or by a topic that normalizes differently than its stored spelling. Every - digest-stemmed or slug-family candidate is validated against the - record it actually holds, and a slug lookup answers only when the - lookup string is itself a slug with a single claiming record: - default-slug reloads stay working, while colliding topics such as - C++ and C# never merge through the shared slug. + candidate is validated against the record it actually holds: a legacy + file keyed by a slug several topics shared must never answer another + topic's lookup, so a store holding C++ in ``c.md`` raises for C# + instead of handing out the C++ record. A slug lookup answers only + when the lookup string is itself a slug and every claiming record + holds one and the same topic: default-slug reloads stay working + (and so does the transient two-file state an interrupted migration + leaves), while colliding topics such as C++ and C#, or two topics + that merely slugify into the same family, never merge through the + shared slug. """ topics_dir = self._get_topics_directory(session, source_id=source_id) if not topics_dir.is_dir(): @@ -895,34 +914,50 @@ def _existing_topic_path(self, session: AgentSession, *, source_id: str, topic: record = MemoryTopicRecord.from_markdown(candidate_path.read_text(encoding="utf-8"), fallback_topic=topic) if record.topic == wanted: return candidate_path - # 2. An exact filename match: a legacy slug-only file from stores - # written before the digest suffix, or a caller passing a persisted - # stem such as an index entry slug. - legacy_name = _slugify_topic(topic) - if len(legacy_name.encode("utf-8")) + len(".md") > 255: - pass # An overlong legacy name cannot exist on a limit bound filesystem. - else: - exact_path = topics_dir / f"{legacy_name}.md" + slug = _slugify_topic(topic) + # Claimants collect every file whose record claims this slug; a slug + # lookup answers below only when they all hold the same topic. + slug_claimants: list[tuple[Path, str]] = [] + # 2. An exact filename match: a persisted stem (an injective identity + # that names exactly one file, such as the slug an index link + # carries), or a legacy slug-only file. The legacy file is + # validated against the record it holds because the pre digest + # scheme keyed several topics by the same slug; an unvalidated hit + # let a missing topic read, merge into, or delete another topic's + # memories. Otherwise it joins the slug claimants. + if len(slug.encode("utf-8")) + len(".md") <= 255: + exact_path = topics_dir / f"{slug}.md" if exact_path.exists(): - return exact_path + record = MemoryTopicRecord.from_markdown(exact_path.read_text(encoding="utf-8"), fallback_topic=topic) + # A legacy file's heading may spell the topic with a different + # case than the lookup; the pre digest scheme stored one file + # per slug, so case was never a distinguishing identity there. + if _is_digest_stem(exact_path.stem) or record.topic.casefold() == wanted.casefold(): + return exact_path + if record.slug == slug: + slug_claimants.append((exact_path, _normalize_topic(record.topic))) # 3. Digest-suffixed stems sharing the slug. A prefix glob alone proves # nothing (a missing topic must never read, merge into, or delete a # longer unrelated slug), so every candidate is validated against # the record it actually holds. A slug lookup answers only when the - # lookup string is itself a slug and exactly one record claims it: - # default-slug reloads stay working, while a topic name whose - # slugification merely lands in the family (two CJK topics both - # slugifying to ``memory-topic``) cannot borrow a neighbor's file. - slug = _slugify_topic(topic) - slug_holders: list[Path] = [] + # lookup string is itself a slug and all claimants hold one and the + # same topic: a topic name whose slugification merely lands in the + # family (two CJK topics both slugifying to ``memory-topic``) + # cannot borrow a neighbor's file. for path in sorted(topics_dir.glob(f"{slug}-*.md")): record = MemoryTopicRecord.from_markdown(path.read_text(encoding="utf-8"), fallback_topic=topic) if record.topic == wanted: return path if record.slug == slug: - slug_holders.append(path) - if topic == slug and len(slug_holders) == 1: - return slug_holders[0] + slug_claimants.append((path, _normalize_topic(record.topic))) + if topic == slug and slug_claimants and len({claimed for _, claimed in slug_claimants}) == 1: + # Prefer a digest-bearing claimant: it is the stem write_topic + # migrates the record to, so the choice stays deterministic even + # while a legacy file and its digest successor coexist. + for claimant_path, _ in slug_claimants: + if _is_digest_stem(claimant_path.stem): + return claimant_path + return slug_claimants[0][0] return None @staticmethod @@ -960,11 +995,41 @@ def _transcript_file_stem(session_id: str) -> str: return _storage_key_segment(raw_session_id, encoded_prefix=_FILE_HISTORY_ENCODED_SESSION_PREFIX) def list_topics(self, session: AgentSession, *, source_id: str) -> list[MemoryTopicRecord]: - """Return all topic memory files visible from the current owner.""" - topics: list[MemoryTopicRecord] = [] + """Return one record per topic visible from the current owner. + + A migration interrupted between the write and the unlink can leave a + legacy slug-only file next to its digest-stemmed successor, both + holding the same topic. The digest-stemmed record (the stem + ``write_topic`` migrates to) wins and the stale copy is hidden rather + than listed twice; the same applies to a legacy heading spelled with a + different case, which the pre digest scheme could never have kept + apart. Distinct digest-era topics, including topics that differ only + by case, each keep their own record. + """ + records_by_topic: dict[str, MemoryTopicRecord] = {} + digest_casefolded: set[str] = set() topics_directory = self._get_topics_directory(session, source_id=source_id) if not topics_directory.exists(): - return topics + return [] + for topic_path in sorted(topics_directory.glob("*.md")): + if not topic_path.is_file(): + continue + record = MemoryTopicRecord.from_markdown( + topic_path.read_text(encoding="utf-8"), + fallback_topic=topic_path.stem.replace("-", " "), + ) + record.set_on_disk_stem(topic_path.stem) + normalized = _normalize_topic(record.topic) + if _is_digest_stem(topic_path.stem): + digest_casefolded.add(normalized.casefold()) + current = records_by_topic.get(normalized) + if ( + current is not None + and _is_digest_stem(current.storage_stem) + and current.updated_at >= record.updated_at + ): + continue + records_by_topic[normalized] = record for topic_path in sorted(topics_directory.glob("*.md")): if not topic_path.is_file(): continue @@ -973,7 +1038,18 @@ def list_topics(self, session: AgentSession, *, source_id: str) -> list[MemoryTo fallback_topic=topic_path.stem.replace("-", " "), ) record.set_on_disk_stem(topic_path.stem) - topics.append(record) + normalized = _normalize_topic(record.topic) + if _is_digest_stem(topic_path.stem): + continue + if normalized.casefold() in digest_casefolded: + # A stale pre digest copy of a topic that already lives on a + # digest stem: hide it rather than list the topic twice. + continue + current = records_by_topic.get(normalized) + if current is not None and current.updated_at >= record.updated_at: + continue + records_by_topic[normalized] = record + topics = list(records_by_topic.values()) return sorted(topics, key=lambda record: (record.topic.lower(), record.updated_at)) def get_topic(self, session: AgentSession, *, source_id: str, topic: str) -> MemoryTopicRecord: @@ -1012,52 +1088,73 @@ def _writable_topic_path(self, session: AgentSession, *, source_id: str, record: def write_topic(self, session: AgentSession, record: MemoryTopicRecord, *, source_id: str) -> None: """Persist one topic memory file. - The digest-stemmed file is the single authoritative record: when this - topic previously lived at another path (a legacy slug-only file, an - extended stem freed by a neighbor's deletion, or a stem written under a - different slug override), that file is retired so lookups can never - surface stale content or resurrect the old record. + The digest-stemmed file is the single authoritative record: after the + write, every other file holding this topic is retired. Resolution + alone cannot drive that cleanup, because once the digest file exists + the lookup already selects it and never names the stale copy again, so + a migration interrupted between the write and the unlink (or one whose + unlink failed outright) would otherwise leave two live records that + ``list_topics`` duplicates and a later delete resurrects. """ - previous_path = self._existing_topic_path(session, source_id=source_id, topic=record.topic) topic_path = self._writable_topic_path(session, source_id=source_id, record=record) _atomic_write_text(topic_path, f"{record.to_markdown()}\n") record.set_on_disk_stem(topic_path.stem) - if ( - previous_path is not None - and previous_path != topic_path - and self._path_holds_topic(previous_path, record.topic) - ): - # A resolution through a legacy slug-only file or a shared slug can - # name another topic's record; retiring it would silently destroy - # that topic, so only a file holding this very topic is retired. - previous_path.unlink(missing_ok=True) + self._retire_stale_topic_files(session, source_id=source_id, topic=record.topic, keep=topic_path) def delete_topic(self, session: AgentSession, *, source_id: str, topic: str) -> None: """Delete one topic memory file. Deletion accepts every alias get_topic answers: the topic name, the - readable slug, and the persisted stem an index link carries. The lookup - resolution itself rejects an ambiguous slug and validates digest stems, - so the file it names is the one record the lookup resolves to; the - topic ownership guard stays on the write path, where a resolution can - name another topic's file for retirement. + readable slug, and the persisted stem an index link carries. The + resolution yields the record the lookup belongs to, then every file + holding that record's topic is removed, so a migration that left both + the digest-stemmed file and its legacy predecessor cannot resurrect + the topic from the stale copy. """ topic_path = self._existing_topic_path(session, source_id=source_id, topic=topic) if topic_path is None: raise FileNotFoundError(f"No memory topic named '{topic}' was found for this owner.") - topic_path.unlink() + record = MemoryTopicRecord.from_markdown(topic_path.read_text(encoding="utf-8"), fallback_topic=topic) + self._retire_stale_topic_files(session, source_id=source_id, topic=record.topic, keep=None) + + def _retire_stale_topic_files( + self, session: AgentSession, *, source_id: str, topic: str, keep: Path | None + ) -> None: + """Unlink every file whose record holds ``topic``, except ``keep``. + + Each occupant is validated before unlinking, so a slug shared by + several topics can never cost a neighbour its file; only this very + topic's stale copies (a legacy slug-only file, an extended stem freed + by a neighbor's deletion, a stem written under a different slug + override) are retired. + """ + topics_dir = self._get_topics_directory(session, source_id=source_id) + if not topics_dir.is_dir(): + return + for path in sorted(topics_dir.glob("*.md")): + if keep is not None and path == keep: + continue + if self._path_holds_topic(path, topic): + path.unlink() def _path_holds_topic(self, path: Path, topic: str) -> bool: """Return whether the record in ``path`` is the given topic's own record. Digest-stemmed resolutions already validated exact identity; this guards the alias-capable resolutions (legacy slug-only files, shared slugs), - where the occupant may be a different topic. The comparison is exact on - the normalized topic: case folding would treat slug shaped neighbours - such as API and api as one topic and retire the original file. + where the occupant may be a different topic. Digest-era comparisons are + exact on the normalized topic: case folding would treat slug shaped + neighbours such as API and api as one topic and retire the original + file. A legacy slug-only file instead folds case, because the pre + digest scheme stored one file per slug, so a heading spelled with a + different case is the same topic and must migrate, not duplicate. """ record = MemoryTopicRecord.from_markdown(path.read_text(encoding="utf-8"), fallback_topic=path.stem) - return _normalize_topic(record.topic) == _normalize_topic(topic) + normalized = _normalize_topic(record.topic) + wanted = _normalize_topic(topic) + if _is_digest_stem(path.stem): + return normalized == wanted + return normalized.casefold() == wanted.casefold() def rebuild_index( self, diff --git a/python/packages/core/tests/core/test_harness_memory.py b/python/packages/core/tests/core/test_harness_memory.py index 09776aa4dea..ff42fe12404 100644 --- a/python/packages/core/tests/core/test_harness_memory.py +++ b/python/packages/core/tests/core/test_harness_memory.py @@ -1265,6 +1265,7 @@ def test_memory_topic_truncation_after_a_separator_stays_canonical(tmp_path) -> assert store.get_topic(session, source_id="src", topic=topic).memories == ["kept"] entries = store.rebuild_index(session, source_id="src", line_limit=10, line_length=200) for entry in entries: + assert entry.storage_stem is not None assert (store._get_topics_directory(session, source_id="src") / f"{entry.storage_stem}.md").exists() assert store.get_topic(session, source_id="src", topic=entry.storage_stem).memories == ["kept"] @@ -1510,7 +1511,7 @@ async def test_memory_consolidation_does_not_lose_a_write_through_an_index_stem( stem = written.storage_stem original_consolidate = provider._consolidate_topic # type: ignore[reportPrivateUsage] - pending_merge: asyncio.Task[None] | None = None + pending_merge: asyncio.Task[MemoryTopicRecord] | None = None async def consolidate_with_a_write_in_flight( *, @@ -1531,7 +1532,7 @@ async def consolidate_with_a_write_in_flight( await asyncio.sleep(0) return record, True - provider._consolidate_topic = consolidate_with_a_write_in_flight # type: ignore[method-assign,reportPrivateUsage] + provider._consolidate_topic = consolidate_with_a_write_in_flight # type: ignore[method-assign,reportPrivateUsage] # ty: ignore[invalid-assignment] await provider._run_consolidation( # type: ignore[reportPrivateUsage] client=_MemoryHarnessClient(), # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] session=session, @@ -1587,13 +1588,21 @@ def get_index_text( def get_transcripts_directory(self, session: AgentSession, *, source_id: str) -> Path: return Path(".") - def search_transcripts(self, session: AgentSession, *, query: str, source_id: str) -> list[dict[str, Any]]: + def search_transcripts( + self, + session: AgentSession, + *, + source_id: str, + query: str, + session_id: str | None = None, + limit: int = 20, + ) -> list[dict[str, Any]]: return [] def read_state(self, session: AgentSession, *, source_id: str) -> dict[str, Any]: return {} - def write_state(self, session: AgentSession, state: dict[str, Any], *, source_id: str) -> None: + def write_state(self, session: AgentSession, state: Mapping[str, Any], *, source_id: str) -> None: pass @@ -1644,19 +1653,10 @@ async def test_memory_consolidation_processes_topics_in_a_conforming_custom_stor source_id=DEFAULT_MEMORY_SOURCE_ID, ) - class _DictClient: - async def get_response(self, *args: Any, **kwargs: Any) -> ChatResponse[Any]: - return ChatResponse( - messages=[ - Message( - role="assistant", - contents=[json.dumps({"summary": "consolidated", "memories": ["m"]})], - ) - ] - ) - count = await provider._run_consolidation( # pyright: ignore[reportPrivateUsage] - client=_DictClient(), + client=_MemoryHarnessClient( # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + consolidation_payload={"summary": "consolidated", "memories": ["m"]} + ), session=session, force=True, now=datetime.now(timezone.utc), @@ -1715,4 +1715,138 @@ def test_memory_file_store_uses_literal_folders_for_safe_identifiers(tmp_path) - assert parts[1] == "user_alice" +def _seed_legacy_file_holding_another_topic(store: MemoryFileStore, session: AgentSession) -> Path: + """Seed a legacy ``c.md`` holding the C++ record, as pre digest stores wrote.""" + topics_dir = store._get_topics_directory(session, source_id=DEFAULT_MEMORY_SOURCE_ID) + topics_dir.mkdir(parents=True, exist_ok=True) + legacy = topics_dir / "c.md" + legacy.write_text( + "# C++\n\nUpdated: t\nSessions: -\n\n## Summary\ncpp\n\n## Memories\n- smart pointers\n", + encoding="utf-8", + ) + return legacy + + +async def test_memory_merge_creates_a_distinct_topic_when_a_legacy_slug_file_holds_another_topic(tmp_path) -> None: + """A legacy slug-only file must not answer a different topic's merge. + + Reading ``C#`` through the legacy ``c.md`` that holds ``C++`` merged the + new memory into the C++ record and wrote it back under the C++ stem, so + the C# topic was never created and its lookup kept returning the C++ + record. + """ + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + legacy = _seed_legacy_file_holding_another_topic(store, session) + + provider = MemoryContextProvider(store=store) + await provider._merge_memory( # pyright: ignore[reportPrivateUsage] + session=session, topic="C#", memory="ctors", now=datetime.now(timezone.utc) + ) + + cpp = store.get_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="C++") + assert cpp.memories == ["smart pointers"] + cs = store.get_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="C#") + assert cs.memories == ["ctors"] + assert cs.storage_stem != legacy.stem + assert legacy.exists() + + +def _seed_dual_file_topic(store: MemoryFileStore, session: AgentSession) -> Path: + """Leave a topic on both the digest stem and a legacy slug-only file. + + This is the state an interrupted migration leaves behind: the digest + write completed, the unlink of the old file did not. + """ + store.write_topic( + session, + MemoryTopicRecord(topic="Travel Preferences", summary="s", memories=["kept"], updated_at="t"), + source_id=DEFAULT_MEMORY_SOURCE_ID, + ) + topics_dir = store._get_topics_directory(session, source_id=DEFAULT_MEMORY_SOURCE_ID) + digest_path = next(path for path in topics_dir.glob("*.md")) + legacy = topics_dir / "travel-preferences.md" + legacy.write_text(digest_path.read_text(encoding="utf-8"), encoding="utf-8") + return digest_path + + +def test_memory_lists_one_record_while_a_migration_left_two_files(tmp_path) -> None: + """A leftover legacy copy must not duplicate the topic in listings.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + digest_path = _seed_dual_file_topic(store, session) + topics_dir = store._get_topics_directory(session, source_id=DEFAULT_MEMORY_SOURCE_ID) + + listed = store.list_topics(session, source_id=DEFAULT_MEMORY_SOURCE_ID) + assert [record.topic for record in listed] == ["Travel Preferences"] + assert listed[0].storage_stem == digest_path.stem + entries = store.rebuild_index(session, source_id=DEFAULT_MEMORY_SOURCE_ID, line_limit=10, line_length=200) + assert len(entries) == 1 + assert entries[0].storage_stem is not None + assert (topics_dir / f"{entries[0].storage_stem}.md").exists() + + +def test_memory_delete_removes_every_file_left_by_an_interrupted_migration(tmp_path) -> None: + """Deleting a topic removes the stale legacy copy, so the topic cannot resurrect.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + _seed_dual_file_topic(store, session) + topics_dir = store._get_topics_directory(session, source_id=DEFAULT_MEMORY_SOURCE_ID) + + store.delete_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="Travel Preferences") + + assert not any(topics_dir.glob("*.md")) + with pytest.raises(FileNotFoundError): + store.get_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="Travel Preferences") + + +def test_memory_write_sweeps_a_leftover_legacy_file_after_a_failed_unlink(tmp_path, monkeypatch) -> None: + """A migration whose unlink failed heals on the next write of the topic.""" + store = MemoryFileStore(tmp_path, owner_state_key="owner_id") + session = AgentSession(session_id="session-1") + session.state["owner_id"] = "alice" + topics_dir = store._get_topics_directory(session, source_id=DEFAULT_MEMORY_SOURCE_ID) + topics_dir.mkdir(parents=True) + legacy = topics_dir / "travel-preferences.md" + legacy.write_text( + "# Travel Preferences\n\nUpdated: t\nSessions: -\n\n## Summary\ns\n\n## Memories\n- kept\n", + encoding="utf-8", + ) + + real_unlink = Path.unlink + + def failing_unlink(self: Path, missing_ok: bool = False) -> None: + if self.name == legacy.name: + raise OSError("simulated crash between write and unlink") + real_unlink(self, missing_ok=missing_ok) + + monkeypatch.setattr(Path, "unlink", failing_unlink) + with pytest.raises(OSError, match="simulated crash"): + store.write_topic( + session, + MemoryTopicRecord(topic="Travel Preferences", summary="s", memories=["kept"], updated_at="t2"), + source_id=DEFAULT_MEMORY_SOURCE_ID, + ) + monkeypatch.undo() + + # The interrupted migration left both files behind, but the stale copy is + # hidden rather than listed as a second topic. + assert legacy.exists() + assert len(store.list_topics(session, source_id=DEFAULT_MEMORY_SOURCE_ID)) == 1 + + # The next write reconciles the directory: only the digest stem remains. + store.write_topic( + session, + MemoryTopicRecord(topic="Travel Preferences", summary="s", memories=["kept", "kept"], updated_at="t3"), + source_id=DEFAULT_MEMORY_SOURCE_ID, + ) + remaining = sorted(path.name for path in topics_dir.glob("*.md")) + assert len(remaining) == 1 + assert remaining[0].startswith("travel-preferences-") + assert store.get_topic(session, source_id=DEFAULT_MEMORY_SOURCE_ID, topic="Travel Preferences").memories == ["kept"] + + # endregion