fix(memory): deleting a memory leaves a similar memory's derived data alone

changes/456.fixed.md

@@ -0,0 +1 @@
+**Deleting a memory no longer removes what Sentient knows from a similar one.** Before, deleting "drives a red Honda City" could also take the MEMORY.md line, journal lines and user model evidence of "sold her red Honda City and now rides a blue Ather". Now only the deleted memory's own lines and evidence go.

docs/API.md

@@ -1773,7 +1773,11 @@ attribute and no word of what it says in common ("a piano recital in July" for "
   earlier wording, chat summaries made of the message it came from that repeat it (with their vectors), the nightly
   journal's lines about it, the user model's evidence from it (an inferred insight with nothing else behind it goes),
   the memory chips on replies and task runs that had it in mind, its USER.md bullet and the MEMORY.md lines that repeat
-  it. The chats themselves stay. The tidy-up's own merges and contradictions don't count as deletes.
+  it. The chats themselves stay. The tidy-up's own merges and contradictions don't count as deletes. The user model's
+  evidence from a memory is found by the memory's id only. Text found by its words (summaries, journal lines, evidence
+  quotes from chats, USER.md and MEMORY.md lines) stays when a memory still in use says it as well or better, or names
+  that memory's id, so deleting one of two similar memories leaves the other's lines alone (#456). The same holds when
+  a memory becomes disputed or superseded and leaves USER.md and MEMORY.md.
 - Migration 0015 gives memories saved before #153 from a messaging app or a device chat their real source (found from
   the review note of a held memory, the `memory_remember` call that saved it, or the one message in such a chat, up to
   30 minutes before it, that says it), so forgetting that app covers them too.

sentient/app.py

@@ -319,11 +319,11 @@ async def delete_chat(self, session_id: str, *, reason: str | None = None) -> No
             log.exception("removing the selected text of chat %s failed", session_id)
         self.bus.publish("session.deleted", {"session_id": session_id, **({"reason": reason} if reason else {})})
 
-    def _after_memory_erased(self, texts: list[str]) -> None:
-        """A memory was deleted (#152): its words leave USER.md and MEMORY.md, and the user model reads its insights
-        again (some may be gone with it)."""
+    def _after_memory_erased(self, erased: Any) -> None:
+        """A memory was deleted (#152): its words leave USER.md and MEMORY.md (not a remaining memory's similar lines,
+        #456), and the user model reads its insights again (some may be gone with it)."""
         try:
-            self.workspace.scrub(texts, self.config.assistant.user_name)
+            self.workspace.scrub(erased.wordings, self.config.assistant.user_name, repeats=erased)
         except OSError:
             log.exception("removing a deleted memory from the workspace files failed")
         model = getattr(self, "user_model", None)

sentient/memory/facts.py

@@ -667,7 +667,7 @@ def __init__(self, store: Store, llm: LLMProvider, config: SentientConfig, bus:
         self.episodic = EpisodicMemory(store, llm, config)
         self._qcache: OrderedDict[tuple[str, str], list[float]] = OrderedDict()
         # set by the app: also scrub the words of erased memories from the workspace files and the user model's
-        # caches (#152); called with the texts that went
+        # caches (#152); called with an ``origins.Erased``: the texts that went, and which lines repeat them (#456)
         self.on_erase: Any = None
 
     async def ensure_schema(self) -> None:
@@ -921,7 +921,9 @@ async def erase(self, fid: int, *, notify: bool = False) -> bool:
         merged copies and source links (:meth:`forget`), the chat summaries made of the messages it came from that
         repeat it, the nightly journal's lines about it, the user model's evidence from it (an inferred insight with
         nothing else behind it goes), the memory chips on replies and task runs that had it in mind, and (through
-        ``on_erase``) the lines of USER.md and MEMORY.md that say it. Its own previous wording goes with it."""
+        ``on_erase``) the lines of USER.md and MEMORY.md that say it. Its own previous wording goes with it. Text found
+        by its words is left alone when a memory still in use says it as well or better (#456): deleting one of two
+        similar memories never strips the other's lines or evidence."""
         await self.ensure_schema()
         row = await self.store.fetchone(
             "SELECT id, content, previous_content, source_ref FROM facts WHERE id = ?", (fid,)
@@ -948,11 +950,17 @@ async def erase(self, fid: int, *, notify: bool = False) -> bool:
             log.exception("cleaning up after memory %s failed", fid)
         return True
 
-    async def _cascade(self, ids: list[int], texts: list[str], refs: list[dict]) -> None:
-        store, user = self.store, self.config.assistant.user_name
+    async def _erased(self, texts: list[str], ids: list[int]) -> origins.Erased:
+        """What ``texts`` said, against every memory still in use (#456): what derived data repeats them goes,
+        unless a remaining memory says it as well or better (:class:`origins.Erased`)."""
+        rows = await self.store.fetchall(f"SELECT id, content FROM facts WHERE {IN_USE}")
+        gone = set(ids)
+        remaining = [(int(r["id"]), str(r["content"])) for r in rows if int(r["id"]) not in gone]
+        return origins.Erased(texts, remaining, self.config.assistant.user_name)
 
-        def repeats(text: str) -> bool:
-            return any(origins.says(t, text or "", user) for t in texts)
+    async def _cascade(self, ids: list[int], texts: list[str], refs: list[dict]) -> None:
+        store = self.store
+        repeats = await self._erased(texts, ids)
 
         # chat summaries made of the messages it came from, that repeat it, and their vectors
         said_in = {str(r["message_id"]) for r in refs if r.get("message_id")}
@@ -972,7 +980,7 @@ def repeats(text: str) -> bool:
                     for line in lines]
             if kept != lines:
                 await store.execute("UPDATE dreams SET journal_md = ? WHERE id = ?", ("\n".join(kept), r["id"]))
-        # the user model's evidence from it
+        # the user model's evidence from it: a memory's by its id only, a quote of a chat or summary when it says it
         wanted = {str(i) for i in ids}
         try:
             insights = await store.fetchall("SELECT rid, id, source, evidence FROM user_insights")
@@ -986,7 +994,7 @@ def repeats(text: str) -> bool:
             if not isinstance(evidence, list):
                 continue
             kept = [e for e in evidence if not (isinstance(e, dict) and (
-                (e.get("kind") == "fact" and str(e.get("ref")) in wanted) or (e.get("quote") and repeats(e["quote"]))
+                str(e.get("ref")) in wanted if e.get("kind") == "fact" else bool(e.get("quote")) and repeats(e["quote"])
             ))]
             if len(kept) == len(evidence):
                 continue
@@ -1017,17 +1025,17 @@ def repeats(text: str) -> bool:
                             f"UPDATE {table} SET memory_sources = ? WHERE rowid = ?", (json.dumps(kept), r["rowid"])
                         )
         if self.on_erase is not None:
-            res = self.on_erase(texts)
+            res = self.on_erase(repeats)
             if hasattr(res, "__await__"):
                 await res
 
-    async def _out_of_prompts(self, texts: list[str]) -> None:
+    async def _out_of_prompts(self, texts: list[str], ids: list[int]) -> None:
         """A memory left the prompts (disputed or superseded): its lines leave USER.md and MEMORY.md too, which go
-        into every prompt (``on_erase``)."""
+        into every prompt (``on_erase``), unless a memory still in use says them as well (#456)."""
         if self.on_erase is None:
             return
         try:
-            res = self.on_erase(texts)
+            res = self.on_erase(await self._erased(texts, ids))
             if hasattr(res, "__await__"):
                 await res
         except Exception:
@@ -1055,7 +1063,7 @@ async def set_state(self, fid: int, state: str) -> dict:
             await self.store.execute(
                 "UPDATE facts SET state = 'disputed', updated_at = ? WHERE id = ?", (now_iso(), fid)
             )
-            await self._out_of_prompts([row["content"]])
+            await self._out_of_prompts([row["content"]], [fid])
         if row["state"] != state:
             self.publish("UPDATE", fid, row["content"], state=state, reason="confirmed" if state == "current" else "disputed")
         fact = await self.get_fact(fid)
@@ -2104,7 +2112,7 @@ async def supersede(self, keep_id: int, drop_id: int) -> bool:
             (keep_id, ts, drop_id),
         )
         await self.vec.delete(drop_id)
-        await self._out_of_prompts([drop["content"]])
+        await self._out_of_prompts([drop["content"]], [drop_id])
         self.publish("DELETE", drop_id, drop["content"], reason="contradicted", superseded_by=keep_id)
         return True
 

sentient/memory/origins.py

@@ -183,14 +183,69 @@ def quote(text: str, fact: str) -> str:
     return best if len(best) <= QUOTE_CHARS else best[: QUOTE_CHARS - 1].rstrip() + "…"
 
 
+def _share(want: set[str], have: set[str]) -> float:
+    if len(want) < 2:
+        return 0.0
+    shared = len(want & have)
+    return shared / len(want) if shared >= max(2, -(-len(want) * 3 // 5)) else 0.0
+
+
+def match(fact: str, text: str, user_name: str = "") -> float:
+    """How fully ``text`` repeats ``fact``: the share of the fact's words it has (besides the user's own name) when
+    it :func:`says` the fact, else 0."""
+    return _share(_words(fact) - _words(user_name), _words(text))
+
+
 def says(fact: str, text: str, user_name: str = "") -> bool:
     """Whether ``text`` repeats ``fact``: most of its words (three in five, at least two), besides the user's own
-    name. Stricter than :func:`backs`: it decides what is scrubbed when a memory is deleted."""
-    want = _words(fact) - _words(user_name)
-    if len(want) < 2:
-        return False
-    have = _words(text)
-    return len(want & have) >= max(2, -(-len(want) * 3 // 5))
+    name. Stricter than :func:`backs`: it decides what is scrubbed when a memory is deleted (with :class:`Erased`)."""
+    return match(fact, text, user_name) > 0
+
+
+_QUOTED = re.compile(r'"([^"\n]+)"|“([^”\n]+)”')
+# a reference to a memory by its id, as derived text writes one: "(memory 7)", "(id: 7)", "[fact #7]"
+_NAMES_ID = re.compile(r"[(\[]\s*(?:id|memory|fact)\s*[:#]?\s*(\d+)\s*[)\]]", re.IGNORECASE)
+_NUMBER = re.compile(r"\d+")
+
+
+class Erased:
+    """What deleted memories said, weighed against the memories that remain (#456).
+
+    Called with a line of derived text (a MEMORY.md bullet, a journal line, a summary, an evidence quote), it says
+    whether the line repeats a deleted wording (:func:`says`) and so goes. A line stays when a remaining current
+    memory says it as well or better, or when it names a remaining memory's id ("(memory 7)", unless the deleted
+    wording it repeats says that number itself): then it is that memory's line, not the deleted one's. Each quoted
+    part of a line is weighed on its own as well, so a journal line like ``I used to remember "A", but now "B"`` goes
+    when A (or B) is deleted, even though B (or A) remains."""
+
+    def __init__(self, texts: list[str], remaining: list[tuple[int, str]], user_name: str = "") -> None:
+        self.user = _words(user_name)
+        self.texts = [t for t in dict.fromkeys(texts) if t]
+        self._gone = [(_words(t) - self.user, set(_NUMBER.findall(t))) for t in self.texts]
+        self._left = [_words(c) - self.user for _, c in remaining]
+        self._ids = {str(i) for i, _ in remaining}
+        self._held = {" ".join(str(c).lower().split()) for _, c in remaining}
+
+    @property
+    def wordings(self) -> list[str]:
+        """The deleted wordings no remaining memory says word for word (an exact USER.md line of one of those is
+        the remaining memory's line too)."""
+        return [t for t in self.texts if " ".join(t.lower().split()) not in self._held]
+
+    def _repeats(self, part: str) -> bool:
+        have = _words(part)
+        said = [(_share(w, have), nums) for w, nums in self._gone]
+        best = max((share for share, _ in said), default=0.0)
+        if best == 0:
+            return False
+        own = {n for share, nums in said if share for n in nums}  # numbers the matching deleted words say
+        if any(m.group(1) in self._ids and m.group(1) not in own for m in _NAMES_ID.finditer(part)):
+            return False
+        return not any(_share(w, have) >= best for w in self._left)
+
+    def __call__(self, text: str) -> bool:
+        text = str(text or "")
+        return any(self._repeats(p) for p in (text, *(a or b for a, b in _QUOTED.findall(text))))
 
 
 def confidence(source: str, *, held: bool = False) -> float:

sentient/memory/workspace.py

@@ -12,6 +12,7 @@
 
 from __future__ import annotations
 
+from collections.abc import Callable
 from datetime import date, timedelta
 from pathlib import Path
 
@@ -171,20 +172,19 @@ def remove_learned(self, facts: list[str], heading: str = "## Learned") -> int:
             self.user.write_text("\n".join(out) + "\n", encoding="utf-8")
         return removed
 
-    def scrub(self, facts: list[str], user_name: str = "") -> int:
+    def scrub(self, facts: list[str], user_name: str = "", repeats: Callable[[str], bool] | None = None) -> int:
         """A memory was deleted (#152): its bullet goes from USER.md's Learned section, and every bullet of MEMORY.md
-        (written from memories by the nightly upkeep) that repeats it (``origins.says``). The user's own text in
-        USER.md is never touched. Returns how many lines went."""
+        (written from memories by the nightly upkeep) that repeats it: ``repeats`` (an ``origins.Erased``, which spares
+        a remaining memory's similar lines, #456), else ``origins.says``. The user's own text in USER.md is never
+        touched. Returns how many lines went."""
         from sentient.memory import origins
 
         removed = self.remove_learned(facts)
-        if not facts or not self.memory.exists():
+        if not self.memory.exists() or not (facts or repeats is not None):
             return removed
+        said = repeats or (lambda line: any(origins.says(f, line, user_name) for f in facts))
         lines = self.memory.read_text(encoding="utf-8", errors="replace").splitlines()
-        kept = [
-            line for line in lines
-            if not (line.lstrip().startswith(("- ", "* ")) and any(origins.says(f, line, user_name) for f in facts))
-        ]
+        kept = [line for line in lines if not (line.lstrip().startswith(("- ", "* ")) and said(line))]
         if len(kept) != len(lines):
             self.memory.write_text("\n".join(kept) + "\n", encoding="utf-8")
         return removed + len(lines) - len(kept)