diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index e6fa66d8221..8978114455f 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -211,7 +211,7 @@ The vector store API is experimental under the shared `VECTOR_STORES` feature ID - **`ContextProvider`** - Base class for context providers (RAG, memory systems) - **`HistoryProvider`** - Base class for conversation history storage - **`InMemoryHistoryProvider`** - Built-in session-state history provider for local runs -- **`FileHistoryProvider`** - Experimental append-only file-backed history provider; msgspec JSON Lines is the default and `serialization_format="msgpack"` uses length-prefixed binary MessagePack records. Custom `dumps`/`loads` remain as deprecated JSON-only compatibility hooks and emit `DeprecationWarning` when supplied. +- **`FileHistoryProvider`** - Experimental append-only file-backed history provider; msgspec JSON Lines is the default and `serialization_format="msgpack"` uses length-prefixed binary MessagePack records. Custom `dumps`/`loads` remain as deprecated JSON-only compatibility hooks and emit `DeprecationWarning` when supplied. A save whose incoming prefix covers the whole stored transcript also persists changed payloads for those stored messages, so in-place compaction annotations reach disk; the rewrite goes through a sibling temporary file and `os.replace`, and a rejected record leaves the previous file untouched. A shorter suffix replay still only appends. - **Mixed computer/function workflow history** - The default `HistoryProvider.after_run` defers completed local function results for loadable providers that store inputs until the computer reply arrives, so history records them once in call order. Custom `after_run` implementations must handle this themselves. diff --git a/python/packages/core/agent_framework/_sessions.py b/python/packages/core/agent_framework/_sessions.py index 1e43f0ca1d9..d19aed7ae08 100644 --- a/python/packages/core/agent_framework/_sessions.py +++ b/python/packages/core/agent_framework/_sessions.py @@ -22,6 +22,7 @@ import logging import math import os +import stat import threading import time import uuid @@ -35,7 +36,7 @@ from functools import partial from hashlib import sha256 from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeAlias, TypeVar, cast +from typing import TYPE_CHECKING, Any, BinaryIO, ClassVar, Literal, TypeAlias, TypeVar, cast import msgspec from typing_extensions import TypedDict @@ -183,6 +184,31 @@ def _get_message_hash(message: Message) -> MessageIdentity: return get_message_identity(message) +def _same_stored_payload(left: Any, right: Any) -> bool: + """Return whether two serialized payloads are identical, including value types. + + Plain ``==`` treats ``1`` and ``True`` (and ``1.0``) as equal, but JSON and + MessagePack store them differently, so a type-only change has to be persisted. + """ + if type(left) is not type(right): + return False + if isinstance(left, Mapping): + left_mapping = cast(Mapping[str, Any], left) + right_mapping = cast(Mapping[str, Any], right) + return left_mapping.keys() == right_mapping.keys() and all( + _same_stored_payload(left_mapping[key], right_mapping[key]) for key in left_mapping + ) + if isinstance(left, list): + left_items = cast(list[Any], left) + right_items = cast(list[Any], right) + return len(left_items) == len(right_items) and all( + _same_stored_payload(a, b) for a, b in zip(left_items, right_items, strict=True) + ) + if isinstance(left, float) and math.isnan(left): + return math.isnan(right) + return bool(left == right) + + def filter_new_messages(existing: Sequence[Message], incoming: Sequence[Message]) -> list[Message]: """Return messages after the ordered overlap with persisted history. @@ -2583,6 +2609,10 @@ class FileHistoryProvider(HistoryProvider): The custom ``dumps`` and ``loads`` constructor arguments remain available for JSON Lines compatibility but are deprecated. + New messages append. When a save replays the whole stored transcript with + changed payloads, for example after compaction annotated stored messages in + place, those records are updated too through an atomic replacement. + Security posture: Persisted history is stored as plaintext JSONL or binary MessagePack on the local filesystem. @@ -2716,7 +2746,7 @@ async def save_messages( state: dict[str, Any] | None = None, **kwargs: Any, ) -> None: - """Append messages to the session's history file.""" + """Append messages to the session's history file, persisting changed stored payloads.""" mark_feature_used(FeatureIndex.CORE_FILE_HISTORY_PROVIDER) del state, kwargs if not messages: @@ -2731,14 +2761,23 @@ def _append_messages() -> None: if self.serialization_format == "json": existing_messages = self._read_json_messages(file_path) if file_path.exists() else [] new_messages = filter_new_messages(existing_messages, messages) - if new_messages: - with file_path.open("a", encoding="utf-8") as file_handle: - for message in new_messages: - file_handle.write(f"{self._serialize_json_message(message)}\n") + updated = self._updated_stored_messages(existing_messages, messages, new_messages) + if not new_messages and not updated: + return + if updated: + self._rewrite_json_messages(file_path, updated, new_messages) + return + with file_path.open("a", encoding="utf-8") as file_handle: + for message in new_messages: + file_handle.write(f"{self._serialize_json_message(message)}\n") return existing_messages = self._read_msgpack_messages(file_path) if file_path.exists() else [] new_messages = filter_new_messages(existing_messages, messages) - if not new_messages: + updated = self._updated_stored_messages(existing_messages, messages, new_messages) + if not new_messages and not updated: + return + if updated: + self._rewrite_msgpack_messages(file_path, updated, new_messages) return # Check the whole batch first so a record the reader would reject never reaches # the file and the existing history stays loadable. Records are encoded into one @@ -2764,6 +2803,86 @@ def _append_messages() -> None: async with async_lock: await asyncio.to_thread(_append_messages) + @staticmethod + def _updated_stored_messages( + existing: Sequence[Message], incoming: Sequence[Message], new_messages: Sequence[Message] + ) -> list[Message] | None: + """Return the transcript with updated stored payloads, or ``None`` when nothing changed. + + Compaction excludes stored messages by annotating them in place, so the same + transcript can arrive again with updated ``additional_properties`` on messages + that are already on disk. ``filter_new_messages`` only reports the unseen tail, + which would leave those annotations unpersisted. + + Only an exact overlap with the stored transcript is considered: the incoming + prefix must cover the whole stored history, so a partial replay that matched a + shorter suffix never rewrites records it did not carry. Payloads are compared as + serialized dicts, which is what the reader reconstructs. + """ + if not existing: + return None + stored_count = len(incoming) - len(new_messages) + if stored_count != len(existing): + return None + merged = list(existing) + changed = False + for index, message in enumerate(incoming[:stored_count]): + if not _same_stored_payload(message.to_dict(), existing[index].to_dict()): + merged[index] = message + changed = True + return merged if changed else None + + def _rewrite_json_messages( + self, file_path: Path, merged: Sequence[Message], new_messages: Sequence[Message] + ) -> None: + """Replace the transcript atomically so a failed encode leaves the old file intact.""" + serialized = "".join( + f"{self._serialize_json_message(message)}\n" for message in [*merged, *new_messages] + ).encode("utf-8") + with self._staging_history_file(file_path) as stage: + stage.write(serialized) + + def _rewrite_msgpack_messages( + self, file_path: Path, merged: Sequence[Message], new_messages: Sequence[Message] + ) -> None: + """Replace the transcript atomically after every record passes the reader's checks.""" + record = bytearray() + # Encode into a sibling staging file so a rejected record never touches the history. + with self._staging_history_file(file_path) as stage: + for message in [*merged, *new_messages]: + self._encode_msgpack_record(message, record) + stage.write(len(record).to_bytes(self._MSGPACK_RECORD_HEADER_BYTES, "big")) + stage.write(record) + + @staticmethod + @contextlib.contextmanager + def _staging_history_file(file_path: Path) -> Generator[BinaryIO]: + """Yield a sibling staging file that replaces ``file_path`` on clean exit. + + The name carries only a short prefix and a random token: a session ID may + already use most of the filesystem's component budget, so embedding the + history filename would push the staging name past it. + + The staging file is created with the history's own access bits (or ``0600`` + when it does not exist yet) so replacing a restricted transcript never + widens who can read it, regardless of the process umask. + """ + stage_path = file_path.with_name(f".history-{uuid.uuid4().hex}.tmp") + try: + mode = stat.S_IMODE(file_path.stat().st_mode) + except FileNotFoundError: + mode = 0o600 + descriptor = os.open(stage_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode) + try: + with os.fdopen(descriptor, "wb") as stage: + yield stage + # os.open applies the umask, so set the intended bits explicitly. + os.chmod(stage_path, mode) + os.replace(stage_path, file_path) + finally: + with contextlib.suppress(FileNotFoundError): + stage_path.unlink() + def _encode_msgpack_record(self, message: Message, record: bytearray) -> None: """Encode ``message`` into ``record`` and reject it if the reader would refuse the record.""" _DEFAULT_MSGPACK_ENCODER.encode_into(message.to_dict(), record) diff --git a/python/packages/core/tests/core/test_sessions.py b/python/packages/core/tests/core/test_sessions.py index 6ffdbb36511..378743a9e99 100644 --- a/python/packages/core/tests/core/test_sessions.py +++ b/python/packages/core/tests/core/test_sessions.py @@ -4,6 +4,7 @@ import json import math import os +import stat import threading import time import tracemalloc @@ -2384,6 +2385,213 @@ async def test_save_messages_preserves_repeated_user_turn_across_saves( loaded = await provider.get_messages("s1") assert [message.text for message in loaded] == ["yes", "first reply", "yes", "second reply"] + @pytest.mark.parametrize("serialization_format", ["json", "msgpack"]) + async def test_save_messages_persists_updated_annotations_for_stored_messages( + self, tmp_path: Path, serialization_format: Literal["json", "msgpack"] + ) -> None: + """Annotations added to an already-stored message reach the file. + + Compaction excludes stored messages by annotating them, so a provider that + only appends unseen messages would keep the exclusion in memory for the + in-memory provider and drop it here. + """ + provider = FileHistoryProvider(tmp_path, serialization_format=serialization_format) + first = Message(role="user", contents=["question"], message_id="m1") + second = Message(role="assistant", contents=["answer"], message_id="m2") + await provider.save_messages("s1", [first, second]) + + first.additional_properties["_excluded"] = True + first.additional_properties["_exclude_reason"] = "sliding_window" + await provider.save_messages("s1", [first, second]) + + reloaded = FileHistoryProvider(tmp_path, serialization_format=serialization_format) + loaded = await reloaded.get_messages("s1") + assert [message.text for message in loaded] == ["question", "answer"] + assert loaded[0].additional_properties["_excluded"] is True + assert loaded[0].additional_properties["_exclude_reason"] == "sliding_window" + assert "_excluded" not in loaded[1].additional_properties + + @pytest.mark.parametrize("serialization_format", ["json", "msgpack"]) + async def test_save_messages_skips_rewrite_when_nothing_changed( + self, tmp_path: Path, serialization_format: Literal["json", "msgpack"] + ) -> None: + """A replayed transcript with identical payloads leaves the file untouched.""" + provider = FileHistoryProvider(tmp_path, serialization_format=serialization_format) + messages = [ + Message(role="user", contents=["question"], message_id="m1"), + Message(role="assistant", contents=["answer"], message_id="m2"), + ] + await provider.save_messages("s1", messages) + session_file = provider._session_file_path("s1") + before = await asyncio.to_thread(session_file.read_bytes) + before_mtime = session_file.stat().st_mtime_ns + + await provider.save_messages("s1", messages) + + assert await asyncio.to_thread(session_file.read_bytes) == before + assert session_file.stat().st_mtime_ns == before_mtime + + @pytest.mark.parametrize("serialization_format", ["json", "msgpack"]) + async def test_annotation_update_keeps_transcript_order_and_appends_new_messages( + self, tmp_path: Path, serialization_format: Literal["json", "msgpack"] + ) -> None: + """An updated message keeps its position while later messages still append.""" + provider = FileHistoryProvider(tmp_path, serialization_format=serialization_format) + first = Message(role="user", contents=["one"], message_id="m1") + second = Message(role="assistant", contents=["two"], message_id="m2") + await provider.save_messages("s1", [first, second]) + + second.additional_properties["_excluded"] = True + third = Message(role="user", contents=["three"], message_id="m3") + await provider.save_messages("s1", [first, second, third]) + + loaded = await FileHistoryProvider(tmp_path, serialization_format=serialization_format).get_messages("s1") + assert [message.text for message in loaded] == ["one", "two", "three"] + assert [message.message_id for message in loaded] == ["m1", "m2", "m3"] + assert loaded[1].additional_properties["_excluded"] is True + + @pytest.mark.parametrize("serialization_format", ["json", "msgpack"]) + async def test_annotation_update_uses_the_existing_dedup_contract( + self, tmp_path: Path, serialization_format: Literal["json", "msgpack"] + ) -> None: + """Content-identified repeats stay separate records rather than collapsing.""" + provider = FileHistoryProvider(tmp_path, serialization_format=serialization_format) + await provider.save_messages( + "s1", [Message(role="user", contents=["yes"]), Message(role="assistant", contents=["first"])] + ) + # A distinct turn with the same user text: the overlap contract appends it. + repeated = Message(role="user", contents=["yes"]) + repeated.additional_properties["_excluded"] = True + await provider.save_messages("s1", [repeated, Message(role="assistant", contents=["second"])]) + + loaded = await FileHistoryProvider(tmp_path, serialization_format=serialization_format).get_messages("s1") + assert [message.text for message in loaded] == ["yes", "first", "yes", "second"] + assert loaded[2].additional_properties["_excluded"] is True + assert "_excluded" not in loaded[0].additional_properties + + @pytest.mark.parametrize("serialization_format", ["json", "msgpack"]) + async def test_annotation_update_rolls_back_when_a_new_record_is_rejected( + self, tmp_path: Path, serialization_format: Literal["json", "msgpack"] + ) -> None: + """A rejected batch restores the previous file, including the pending annotation.""" + provider = FileHistoryProvider(tmp_path, serialization_format=serialization_format) + first = Message(role="user", contents=["question"], message_id="m1") + await provider.save_messages("s1", [first]) + session_file = provider._session_file_path("s1") + before = await asyncio.to_thread(session_file.read_bytes) + + first.additional_properties["_excluded"] = True + oversized = Message(role="assistant", contents=["x" * 4096], message_id="m2") + with ( + patch.object( + FileHistoryProvider, + "_encode_msgpack_record" if serialization_format == "msgpack" else "_serialize_json_message", + side_effect=ValueError("record rejected"), + ), + pytest.raises(ValueError, match="record rejected"), + ): + await provider.save_messages("s1", [first, oversized]) + + assert await asyncio.to_thread(session_file.read_bytes) == before + loaded = await FileHistoryProvider(tmp_path, serialization_format=serialization_format).get_messages("s1") + assert [message.text for message in loaded] == ["question"] + assert "_excluded" not in loaded[0].additional_properties + + @pytest.mark.parametrize("serialization_format", ["json", "msgpack"]) + async def test_partial_replay_does_not_rewrite_records_it_did_not_carry( + self, tmp_path: Path, serialization_format: Literal["json", "msgpack"] + ) -> None: + """A replay covering only the stored suffix appends without touching earlier records.""" + provider = FileHistoryProvider(tmp_path, serialization_format=serialization_format) + first = Message(role="user", contents=["one"], message_id="m1") + second = Message(role="assistant", contents=["two"], message_id="m2") + await provider.save_messages("s1", [first, second]) + session_file = provider._session_file_path("s1") + before = await asyncio.to_thread(session_file.read_bytes) + + # Only the last stored message is carried, with a changed payload. + changed_second = Message(role="assistant", contents=["two"], message_id="m2") + changed_second.additional_properties["_excluded"] = True + third = Message(role="user", contents=["three"], message_id="m3") + await provider.save_messages("s1", [changed_second, third]) + + after = await asyncio.to_thread(session_file.read_bytes) + assert after.startswith(before) + loaded = await FileHistoryProvider(tmp_path, serialization_format=serialization_format).get_messages("s1") + assert [message.text for message in loaded] == ["one", "two", "three"] + assert "_excluded" not in loaded[1].additional_properties + + @pytest.mark.parametrize("serialization_format", ["json", "msgpack"]) + async def test_annotation_update_preserves_restricted_file_permissions( + self, tmp_path: Path, serialization_format: Literal["json", "msgpack"] + ) -> None: + """Replacing a 0600 transcript must not widen it, whatever the process umask is.""" + provider = FileHistoryProvider(tmp_path, serialization_format=serialization_format) + message = Message(role="user", contents=["question"], message_id="m1") + await provider.save_messages("s1", [message]) + session_file = provider._session_file_path("s1") + await asyncio.to_thread(session_file.chmod, 0o600) + + previous_umask = os.umask(0o022) + try: + message.additional_properties["_excluded"] = True + await provider.save_messages("s1", [message]) + finally: + os.umask(previous_umask) + + assert stat.S_IMODE(session_file.stat().st_mode) == 0o600 + loaded = await FileHistoryProvider(tmp_path, serialization_format=serialization_format).get_messages("s1") + assert loaded[0].additional_properties["_excluded"] is True + + @pytest.mark.parametrize("serialization_format", ["json", "msgpack"]) + async def test_annotation_update_persists_type_only_payload_change( + self, tmp_path: Path, serialization_format: Literal["json", "msgpack"] + ) -> None: + """``1`` and ``True`` are equal in Python but distinct on disk, so the change persists.""" + provider = FileHistoryProvider(tmp_path, serialization_format=serialization_format) + message = Message(role="user", contents=["question"], message_id="m1") + message.additional_properties["flag"] = 1 + await provider.save_messages("s1", [message]) + + message.additional_properties["flag"] = True + await provider.save_messages("s1", [message]) + + loaded = await FileHistoryProvider(tmp_path, serialization_format=serialization_format).get_messages("s1") + assert loaded[0].additional_properties["flag"] is True + + @pytest.mark.parametrize("serialization_format", ["json", "msgpack"]) + async def test_replay_with_nan_payload_does_not_rewrite( + self, tmp_path: Path, serialization_format: Literal["json", "msgpack"] + ) -> None: + """NaN never equals itself, so an unchanged NaN payload must not force a full rewrite.""" + provider = FileHistoryProvider(tmp_path, serialization_format=serialization_format) + message = Message(role="user", contents=["question"], message_id="m1") + message.additional_properties["score"] = math.nan + await provider.save_messages("s1", [message]) + session_file = provider._session_file_path("s1") + before_inode = session_file.stat().st_ino + + await provider.save_messages("s1", [message]) + + assert session_file.stat().st_ino == before_inode + + @pytest.mark.parametrize("serialization_format", ["json", "msgpack"]) + async def test_annotation_update_works_for_long_session_ids( + self, tmp_path: Path, serialization_format: Literal["json", "msgpack"] + ) -> None: + """A session ID the append path accepts must also support the rewrite.""" + provider = FileHistoryProvider(tmp_path, serialization_format=serialization_format) + session_id = "a" * 240 + message = Message(role="user", contents=["question"], message_id="m1") + await provider.save_messages(session_id, [message]) + + message.additional_properties["_excluded"] = True + await provider.save_messages(session_id, [message]) + + loaded = await FileHistoryProvider(tmp_path, serialization_format=serialization_format).get_messages(session_id) + assert loaded[0].additional_properties["_excluded"] is True + assert not await asyncio.to_thread(lambda: list(tmp_path.glob(".history-*.tmp"))) + # --------------------------------------------------------------------------- # Run-persistence gate tests