diff --git a/changes/441.added.md b/changes/441.added.md new file mode 100644 index 00000000..b6f8369c --- /dev/null +++ b/changes/441.added.md @@ -0,0 +1,3 @@ +**Old versions of memories are cleaned up after a while.** When Sentient replaces a memory with a newer one, the old +version is now deleted for good after 180 days, together with everything made from it. Change this in Settings, or set +it to 0 to keep old versions forever. Your current memories are never touched. diff --git a/desktop/src/features/settings/SchemaForm.tsx b/desktop/src/features/settings/SchemaForm.tsx index 2b9ae938..137e3e98 100644 --- a/desktop/src/features/settings/SchemaForm.tsx +++ b/desktop/src/features/settings/SchemaForm.tsx @@ -53,6 +53,7 @@ export const LABELS: Record = { 'memory.graph_link_similarity': { label: 'Graph link threshold', percent: true }, 'memory.summarize_after_minutes': { label: 'Summarize conversations after', unit: 'min' }, 'memory.summary_chunk_messages': { label: 'Messages per summary', unit: 'messages' }, + 'memory.keep_superseded_days': { label: 'Keep replaced memories for', unit: 'days', description: '0 keeps them forever. Current memories are not affected.' }, // folders: matching pieces of notes that go along with each message (#354) 'folders.prompt_notes': { label: 'Note pieces with each message', unit: 'pieces' }, 'folders.prompt_min_similarity': { label: 'How closely a note must match', percent: true }, diff --git a/docs/API.md b/docs/API.md index 7ca994a3..90986698 100644 --- a/docs/API.md +++ b/docs/API.md @@ -88,7 +88,7 @@ Envelope: `{"type": "task.updated", "data": {...}, "ts": "..."}` | `notification.updated` | full **Notification** after its payload changed (suggestion approved/dismissed, approval answered, task plan approved/declined: a `task` notification with `payload.event = "approval_needed"` gains `payload.status = "approved"\|"declined"`; a task question (`payload.event = "question"`) gains `payload.status = "answered"` with `payload.answer`, or `"cancelled"`) | | `notification.read` / `notification.deleted` | `{id}` (`null` = all) | | `integration.updated` | **Integration** (ยง5) | -| `memory.updated` | `{action: "ADD"\|"UPDATE"\|"DELETE", id, content?, status?: "pending", reason?, previous?, session_id?}`; `status: "pending"` means the memory waits in the review inbox (section 7), with `reason: "instruction"` when it reads like an instruction (#234); `previous` is an UPDATE's words before it and `session_id` the chat a memory was learned in (extraction after a turn, `memory_remember`): that chat shows "Remembered: ... Undo" (Undo is `DELETE /api/memories/{id}`, or `PUT` with `previous` for an update) or "Waiting for your review: ..." with Review (#234); bulk changes (import, delete by source, expiry purge) send `id: null` plus `source?`/`reason?` and `count`. Review sends `reason: "approved"` (ADD, or DELETE with `merged_into` when the same words were already remembered), `"discarded"` (DELETE) and `"review_expired"` (bulk DELETE). Memory scopes (#155, section 7): a save carries `scope` (where it was kept); moving a memory sends UPDATE with `reason: "moved"`, `scope` and `previous_scope`; resetting a scope sends a bulk DELETE with `reason: "scope_reset"`, `scope` and `count`. Forgetting a source (#153) sends a bulk DELETE with `source`, `count` (deleted) and `unlinked` (kept because another source backs them), and `reason: "forgotten"` when it came with a disconnect | +| `memory.updated` | `{action: "ADD"\|"UPDATE"\|"DELETE", id, content?, status?: "pending", reason?, previous?, session_id?}`; `status: "pending"` means the memory waits in the review inbox (section 7), with `reason: "instruction"` when it reads like an instruction (#234); `previous` is an UPDATE's words before it and `session_id` the chat a memory was learned in (extraction after a turn, `memory_remember`): that chat shows "Remembered: ... Undo" (Undo is `DELETE /api/memories/{id}`, or `PUT` with `previous` for an update) or "Waiting for your review: ..." with Review (#234); bulk changes (import, delete by source, expiry purge) send `id: null` plus `source?`/`reason?` and `count`. Superseded-history retention sends a bulk DELETE with `reason: "superseded_expired"` and `count` (the number erased). Review sends `reason: "approved"` (ADD, or DELETE with `merged_into` when the same words were already remembered), `"discarded"` (DELETE) and `"review_expired"` (bulk DELETE). Memory scopes (#155, section 7): a save carries `scope` (where it was kept); moving a memory sends UPDATE with `reason: "moved"`, `scope` and `previous_scope`; resetting a scope sends a bulk DELETE with `reason: "scope_reset"`, `scope` and `count`. Forgetting a source (#153) sends a bulk DELETE with `source`, `count` (deleted) and `unlinked` (kept because another source backs them), and `reason: "forgotten"` when it came with a disconnect | | `memory.forgotten` | `{id, source, label, counts, at}` (`counts`: **ForgetCounts** without `source`): "Also forget what Sentient learned from it" ran when a source was disconnected (#153, section 7); the window refreshes memories, chats and Activity | | `skill.updated` | `{name, state: "active"\|"pending_review"\|"stale"\|"archived"\|"rejected"\|"deleted"}` | | `mcp.review` | `{server, reason?: "new"\|"changed", tools?: [tool names on the server], needed}`: an MCP server's tools wait for review, or the user answered (section 5) | @@ -1771,7 +1771,11 @@ attribute and no word of what it says in common ("a piano recital in July" for " found a newer memory that contradicts it). Only current memories are recalled into chats, tasks, proactivity, the user model, USER.md and MEMORY.md (a memory that becomes disputed or superseded also leaves those two files); dedup still sees disputed ones, so saying one again in a chat makes it current. - A superseded memory stays as history (out of recall, not listed unless `state` asks for it). + A superseded memory stays as history (out of recall, not listed unless `state` asks for it) until + `memory.keep_superseded_days` has elapsed since it was replaced (default 180 days; 0 disables this retention + cleanup, keeping history indefinitely unless explicitly deleted or expired). Changing its scope does not restart + the retention period. Expiry cleanup erases older superseded memories and their derived data through the delete + cascade below; this retention rule never selects current or disputed memories. - Deleting a memory (`DELETE /api/memories/{id}`, the `memory_forget` tool, discarding one from review, resetting a scope, forgetting a source) also removes everything derived from it: its vector, merged copies and source links, its earlier wording, chat summaries made of the message it came from that repeat it (with their vectors), the nightly @@ -1951,7 +1955,8 @@ facts of that source too. Engine: `sentient.memory.review` (`for_context(ctx)`, Engine notes: recall used by the system prompt and `memory_recall` is hybrid (same ranking as `q` above) and counts each recalled fact (`facts.recall_count`, `last_recalled_at`; dreaming promotes often-recalled short-term facts). Saying a memory isn't right or is publishes UPDATE with `state` and `reason: "disputed"|"confirmed"` (#152). Bulk and consolidation changes publish `memory.updated` with `reason` (`merged` | `contradicted` (the older one is superseded, #152) | -`promoted` | `expired`) and, for merges/contradictions, `merged_into` / `superseded_by`. +`promoted` | `expired` | `superseded_expired` (bulk DELETE after superseded-history retention, with `count`)) and, for +merges/contradictions, `merged_into` / `superseded_by`. --- diff --git a/sentient/config/schema.py b/sentient/config/schema.py index c295002d..79ff8e1b 100644 --- a/sentient/config/schema.py +++ b/sentient/config/schema.py @@ -546,6 +546,11 @@ class MemoryConfig(BaseModel): description="Memories waiting for your review (from emails, web pages, imports or work Sentient did on its " "own) are let go after this many days without an answer.", ) + keep_superseded_days: int = Field( + 180, ge=0, le=3650, + description="When a newer memory replaces an older one, the older one is kept as history for this many days, " + "then deleted for good. Your current memories are never deleted by this. 0 keeps the history forever.", + ) # ----------------------------------------------------------------------------- tasks (owner: tasks agent) diff --git a/sentient/memory/facts.py b/sentient/memory/facts.py index 547ac4c7..08814acd 100644 --- a/sentient/memory/facts.py +++ b/sentient/memory/facts.py @@ -926,7 +926,7 @@ async def erase(self, fid: int, *, notify: bool = False) -> bool: similar memories never strips the other's lines or evidence.""" await self.ensure_schema() row = await self.store.fetchone( - "SELECT id, content, previous_content, source_ref FROM facts WHERE id = ?", (fid,) + "SELECT id, content, previous_content, source_ref, state, superseded_by FROM facts WHERE id = ?", (fid,) ) if row is None: return False @@ -941,6 +941,11 @@ async def erase(self, fid: int, *, notify: bool = False) -> bool: ) if not await self.forget(fid, notify=notify): return False + if row["state"] == "superseded" and row["superseded_by"] is not None: + await self.store.execute( + "UPDATE facts SET previous_content = NULL WHERE id = ? AND previous_content = ?", + (row["superseded_by"], row["content"]), + ) rows = [row, *copies] texts = list(dict.fromkeys(t for r in rows for t in (r["content"], r["previous_content"]) if t)) refs = [_load_ref(r["source_ref"]) for r in rows] + [_load_ref(r["ref"]) for r in links] @@ -1226,7 +1231,9 @@ async def set_scope(self, fid: int, scope: str) -> dict: raise KeyError(fid) if row["scope"] != target: await self.store.execute( - "UPDATE facts SET scope = ?, updated_at = ? WHERE id = ?", (target, now_iso(), fid) + "UPDATE facts SET scope = ?, updated_at = CASE WHEN state = 'superseded' THEN updated_at ELSE ? END" + " WHERE id = ?", + (target, now_iso(), fid), ) self.publish("UPDATE", fid, row["content"], scope=target, previous_scope=row["scope"], reason="moved") fact = await self.get_fact(fid) @@ -1921,13 +1928,35 @@ async def graph(self, max_nodes: int = 1500) -> dict: # ------------------------------------------------------------------ housekeeping async def purge_expired(self) -> int: rows = await self.store.fetchall( - "SELECT id FROM facts WHERE expires_at IS NOT NULL AND expires_at < ?", (now_iso(),) + "SELECT id, state FROM facts WHERE expires_at IS NOT NULL AND expires_at < ?", (now_iso(),) ) for r in rows: - await self.forget(int(r["id"])) + # Superseded rows have derived history too; discard it with the same cascade as retention cleanup. + if r["state"] == "superseded": + await self.erase(int(r["id"])) + else: + await self.forget(int(r["id"])) if rows: self.publish("DELETE", None, None, reason="expired", count=len(rows)) - return len(rows) + return len(rows) + await self.purge_superseded() + + async def purge_superseded(self) -> int: + """Erase memories that were superseded more than ``memory.keep_superseded_days`` ago, with everything derived + from them (:meth:`erase`). 0 keeps them forever. Current and disputed memories are never touched.""" + days = self.config.memory.keep_superseded_days + if days <= 0: + return 0 + cutoff = (datetime.now(UTC) - timedelta(days=days)).isoformat() + rows = await self.store.fetchall( + "SELECT id FROM facts WHERE state = 'superseded' AND updated_at < ?", (cutoff,) + ) + erased = 0 + for r in rows: + if await self.erase(int(r["id"])): + erased += 1 + if erased: + self.publish("DELETE", None, None, reason="superseded_expired", count=erased) + return erased async def count(self) -> int: row = await self.store.fetchone(f"SELECT COUNT(*) AS n FROM facts WHERE {ACTIVE}") @@ -2205,7 +2234,7 @@ async def promote_recalled(self, min_recalls: int, limit: int = 50) -> list[dict await self.ensure_schema() rows = await self.store.fetchall( f"SELECT {MEMORY_COLUMNS} FROM facts WHERE {ACTIVE} AND memory_type = 'short-term' AND recall_count >= ?" - " AND (expires_at IS NULL OR expires_at > ?) ORDER BY recall_count DESC LIMIT ?", + " AND state != 'superseded' AND (expires_at IS NULL OR expires_at > ?) ORDER BY recall_count DESC LIMIT ?", (min_recalls, now_iso(), limit), ) out = [] diff --git a/tests/memory/test_superseded_expiry.py b/tests/memory/test_superseded_expiry.py new file mode 100644 index 00000000..2c0490da --- /dev/null +++ b/tests/memory/test_superseded_expiry.py @@ -0,0 +1,175 @@ +"""Superseded memories are history that expires (#441): ``memory.keep_superseded_days``, run by the nightly purge.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta + +import pytest + +from sentient.store.db import now_iso +from tests.memory.helpers import add_fact + + +def _ago(days: int) -> str: + return (datetime.now(UTC) - timedelta(days=days)).isoformat() + + +async def _superseded_pair(app, old_text: str, new_text: str, *, age_days: int) -> tuple[int, int]: + mem = app.memory + old, new = await add_fact(mem, old_text), await add_fact(mem, new_text) + assert await mem.supersede(new, old) + await app.store.execute("UPDATE facts SET updated_at = ? WHERE id = ?", (_ago(age_days), old)) + return old, new + + +async def test_old_superseded_memories_expire_and_current_ones_stay(app): + mem = app.memory + assert app.config.memory.keep_superseded_days == 180 + old, new = await _superseded_pair(app, "Maya lives in Pune", "Maya lives in Bengaluru", age_days=181) + recent, newer = await _superseded_pair(app, "Maya works at Acme", "Maya works at Existence", age_days=10) + disputed = await add_fact(mem, "Maya is left-handed") + await mem.set_state(disputed, "disputed") + await app.store.execute("UPDATE facts SET updated_at = ? WHERE id IN (?, ?, ?)", (_ago(400), new, newer, disputed)) + + assert await mem.purge_expired() == 1 + + assert await mem.get_fact(old) is None + for fid in (new, recent, newer, disputed): + assert await mem.get_fact(fid) is not None + assert (await mem.get_fact(new))["state"] == "current" + + +async def test_zero_keeps_superseded_memories_forever(app): + mem = app.memory + app.config.memory.keep_superseded_days = 0 + old, _ = await _superseded_pair(app, "Maya lives in Pune", "Maya lives in Bengaluru", age_days=5000) + assert await mem.purge_expired() == 0 + assert (await mem.get_fact(old))["state"] == "superseded" + + +async def test_explicit_expiry_still_purges_alongside(app): + mem = app.memory + short = await add_fact(mem, "Maya has a dentist appointment on Friday", memory_type="short-term") + await app.store.execute("UPDATE facts SET expires_at = '2000-01-01T00:00:00+00:00' WHERE id = ?", (short,)) + old, _ = await _superseded_pair(app, "Maya lives in Pune", "Maya lives in Bengaluru", age_days=200) + assert await mem.purge_expired() == 2 + assert await mem.get_fact(short) is None and await mem.get_fact(old) is None + + +async def test_expired_superseded_fact_uses_the_cascade_before_its_retention_limit(app): + mem, store = app.memory, app.store + old, new = await _superseded_pair(app, "Maya lives in Pune", "Maya lives in Bengaluru", age_days=10) + await store.execute("UPDATE facts SET expires_at = '2000-01-01T00:00:00+00:00' WHERE id = ?", (old,)) + await store.execute( + "INSERT INTO dreams(id, started_at, status, trigger, stats, journal_md) VALUES('d1', ?, 'completed', 'manual'," + " '{}', ?)", + (now_iso(), "- I used to remember Maya lives in Pune"), + ) + insight = await app.user_model._insert( + "Maya is from Pune", "personal", confidence=0.6, status="active", source="inferred", + evidence=[{"kind": "fact", "ref": str(old), "quote": "Maya lives in Pune"}], + ) + + assert await mem.purge_expired() == 1 + + assert await mem.get_fact(old) is None and await mem.get_fact(new) is not None + assert await store.fetchall("SELECT id FROM user_insights WHERE id = ?", (insight["id"],)) == [] + journal = (await store.fetchone("SELECT journal_md FROM dreams WHERE id = 'd1'"))["journal_md"] + assert "Pune" not in journal and "(a memory you deleted)" in journal + + +async def test_expiring_a_superseded_memory_cascades(app): + mem, store = app.memory, app.store + old, new = await _superseded_pair(app, "Maya lives in Pune", "Maya lives in Bengaluru", age_days=300) + await mem.link_sources(old, ["file:cv.txt"]) + assert await store.fetchall("SELECT fact_id FROM fact_sources WHERE fact_id = ?", (old,)) + await store.execute( + "INSERT INTO dreams(id, started_at, status, trigger, stats, journal_md) VALUES('d1', ?, 'completed', 'manual'," + " '{}', ?)", + (now_iso(), "Tonight I tidied up.\n- I used to remember \"Maya lives in Pune\"\n- Maya lives in Bengaluru"), + ) + kept = await app.user_model._insert( + "Maya has moved around", "personal", confidence=0.6, status="active", source="inferred", + evidence=[{"kind": "fact", "ref": str(old), "quote": "x"}, {"kind": "fact", "ref": str(new), "quote": "y"}], + ) + gone = await app.user_model._insert( + "Maya is from Pune", "personal", confidence=0.6, status="active", source="inferred", + evidence=[{"kind": "fact", "ref": str(old), "quote": "x"}], + ) + + assert await mem.purge_superseded() == 1 + + assert await mem.get_fact(old) is None + assert await store.fetchall("SELECT rowid FROM facts_vec WHERE rowid = ?", (old,)) == [] + assert await store.fetchall("SELECT fact_id FROM fact_sources WHERE fact_id = ?", (old,)) == [] + journal = (await store.fetchone("SELECT journal_md FROM dreams WHERE id = 'd1'"))["journal_md"] + assert "Pune" not in journal and "Bengaluru" in journal and "(a memory you deleted)" in journal + insights = {r["id"]: json.loads(r["evidence"]) for r in await store.fetchall("SELECT id, evidence FROM user_insights")} + assert gone["id"] not in insights and [e["ref"] for e in insights[kept["id"]]] == [str(new)] + assert await mem.get_fact(new) is not None + + +@pytest.mark.parametrize("expiry", ["retention", "explicit"]) +@pytest.mark.parametrize("preserve_unrelated", [False, True]) +async def test_expiry_clears_only_the_matching_previous_content(app, expiry, preserve_unrelated): + """Expired history must not remain on its successor or erase an unrelated prior version.""" + mem, store = app.memory, app.store + old_text = "Maya lives in Pune" + old, new = await _superseded_pair( + app, old_text, "Maya lives in Bengaluru", age_days=181 if expiry == "retention" else 10 + ) + if preserve_unrelated: + await store.execute("UPDATE facts SET previous_content = 'another earlier version' WHERE id = ?", (new,)) + if expiry == "explicit": + await store.execute("UPDATE facts SET expires_at = '2000-01-01T00:00:00+00:00' WHERE id = ?", (old,)) + + assert await mem.purge_expired() == 1 + + assert await mem.get_fact(old) is None + current = await mem.get_fact(new) + assert current["content"] == "Maya lives in Bengaluru" and current["state"] == "current" + assert current["previous_content"] == ("another earlier version" if preserve_unrelated else None) + + +async def test_moving_superseded_memory_preserves_retention_age(app): + """Moving overdue history to another scope must not restart its retention period.""" + mem = app.memory + old, _ = await _superseded_pair(app, "Maya prefers tea", "Maya prefers coffee", age_days=181) + superseded_at = (await mem.get_fact(old))["updated_at"] + + moved = await mem.set_scope(old, "work") + + assert moved["scope"] == "work" and moved["updated_at"] == superseded_at + assert await mem.purge_expired() == 1 + assert await mem.get_fact(old) is None + + +@pytest.mark.parametrize("state", ["current", "disputed"]) +async def test_moving_other_memory_states_still_updates_timestamp(app, state): + """The timestamp preservation exception must be limited to superseded history.""" + mem = app.memory + fid = await add_fact(mem, "Maya uses a standing desk") + await mem.set_state(fid, state) + before = _ago(181) + await app.store.execute("UPDATE facts SET updated_at = ? WHERE id = ?", (before, fid)) + + moved = await mem.set_scope(fid, "work") + + assert moved["updated_at"] != before and moved["state"] == state + + +async def test_promotion_skips_superseded_memories_so_their_retention_age_stays(app): + """Promotion rewrites ``updated_at``; doing that to superseded history would restart its retention period.""" + mem = app.memory + old = await add_fact(mem, "Maya parks on level 2", memory_type="short-term") + new = await add_fact(mem, "Maya parks on level 3", memory_type="short-term") + assert await mem.supersede(new, old) + await app.store.execute("UPDATE facts SET recall_count = 5, updated_at = ? WHERE id IN (?, ?)", (_ago(181), old, new)) + + promoted = await mem.promote_recalled(min_recalls=3) + + assert [m["id"] for m in promoted] == [new] + assert (await mem.get_fact(old))["memory_type"] == "short-term" + assert await mem.purge_expired() == 1 + assert await mem.get_fact(old) is None