feat(retrieval): compose evidence in Knowhere and keep raw results for debug

apps/api/app/api/v1/routes/retrieval.py

@@ -154,9 +154,13 @@ class RetrievalQueryResponse(BaseModel):
     namespace: str
     query: str
     router_used: str
+    evidence: list[dict] = Field(
+        default_factory=list,
+        description="Composed evidence parts (text and inline images) for downstream agents.",
+    )
     evidence_text: str = Field(
         default="",
-        description="Hierarchical evidence text. Primary output for downstream agents.",
+        description="Text projection of evidence. Tables stay as HTML; images are data URLs.",
     )
     answer_text: str = Field(
         default="",
@@ -166,7 +170,10 @@ class RetrievalQueryResponse(BaseModel):
         ),
     )
     referenced_chunks: list[dict] = Field(default_factory=list)
-    results: list[dict] = Field(default_factory=list)
+    results: list[dict] = Field(
+        default_factory=list,
+        description="Raw path chunks for debug. Content keeps placeholders; composed parts live on evidence.",
+    )
     stop_reason: str | None = None
     failure_reason: str | None = None
     decision_trace: list[dict] | None = Field(

apps/api/app/mcp/retrieval_server.py

@@ -54,16 +54,20 @@ def resolve_mcp_namespace(*, ctx: Context | None) -> str:
 
 
 def to_mcp_query_response(response: dict[str, Any]) -> dict[str, Any]:
-    """Project the internal retrieval response to the MCP agent contract.
+    """Project the public retrieval response to the MCP agent contract.
 
-    MCP returns exactly 3 PRIMARY fields:
-    - evidence_text: hierarchical evidence tree for LLM consumption
+    MCP returns the same Knowhere package the HTTP query emits:
+    - evidence: composed parts (text/HTML and inline images)
+    - evidence_text: text projection of those parts
+    - results: raw path chunks for debug
     - referenced_chunks: structured chunk references for citation / follow-up
     - decision_trace: navigation decisions including terminal stop/failure
     """
     return {
         "query": response.get("query"),
+        "evidence": response.get("evidence") or [],
         "evidence_text": response.get("evidence_text") or "",
+        "results": response.get("results") or [],
         "referenced_chunks": response.get("referenced_chunks") or [],
         "decision_trace": response.get("decision_trace") or [],
     }
@@ -101,10 +105,12 @@ def create_retrieval_mcp_server(
     @server.tool(
         name="retrieval.query",
         description=(
-            "Search published documents. Returns evidence_text (hierarchical "
-            "evidence for LLM consumption), referenced_chunks (cited chunk "
-            "metadata for follow-up queries), and decision_trace (navigation "
-            "decisions including stop/failure reasons). "
+            "Search published documents. Compose already happened in Knowhere. "
+            "Returns evidence (composed parts for LLM consumption), "
+            "results (raw path chunks for debug), "
+            "evidence_text (text projection of evidence), "
+            "referenced_chunks (cited chunk metadata for follow-up queries), "
+            "and decision_trace (navigation decisions including stop/failure reasons). "
             "Include navigation intent directly in your query text — the "
             "engine will automatically locate the right documents and sections."
         ),

apps/api/tests/contract/test_evidence_renderer_contract.py

@@ -1,46 +1,91 @@
-from shared.services.retrieval.execution.routes import _render_rows_evidence
+import asyncio
 
+from app.mcp.retrieval_server import to_mcp_query_response
+from shared.services.retrieval.execution.response_projection import (
+    project_public_retrieval_response,
+)
+from shared.services.retrieval.execution.routes import _evidence_fields
 
-def test_render_rows_evidence_should_group_siblings_under_parent_path() -> None:
+
+def test_evidence_fields_flatten_composed_parts_in_result_order() -> None:
     rows = [
+        {"composed": [{"type": "text", "text": "first"}]},
         {
-            "chunk_id": "c2",
-            "content": "second section content",
-            "sort_order": 2,
-            "source": {
-                "source_file_name": "alpha.pdf",
-                "section_path": "Alpha / Two",
-            },
-        },
-        {
-            "chunk_id": "c1",
-            "content": "first section content\nwith more detail",
-            "sort_order": 1,
-            "source": {
-                "source_file_name": "alpha.pdf",
-                "section_path": "Alpha / One",
-            },
-        },
-        {
-            "chunk_id": "c3",
-            "content": "<table><tr><td>metric</td></tr></table>",
-            "source_file_name": "beta.pdf",
-            "section_path": "Beta / Table",
+            "composed": [
+                {"type": "text", "text": "mid "},
+                {"type": "image", "media_type": "image/png", "data": "abc"},
+            ]
         },
+        {"composed": [{"type": "text", "text": "<table><tr><td>metric</td></tr></table>"}]},
+    ]
+
+    fields = _evidence_fields(rows)
+
+    assert fields["evidence"] == [
+        {"type": "text", "text": "first"},
+        {"type": "text", "text": "mid "},
+        {"type": "image", "media_type": "image/png", "data": "abc"},
+        {"type": "text", "text": "<table><tr><td>metric</td></tr></table>"},
     ]
+    assert fields["evidence_text"] == (
+        "firstmid data:image/png;base64,abc<table><tr><td>metric</td></tr></table>"
+    )
+
+
+def test_mcp_query_response_keeps_evidence_and_debug_results() -> None:
+    response = to_mcp_query_response(
+        {
+            "query": "q",
+            "evidence": [{"type": "text", "text": "t"}],
+            "evidence_text": "t",
+            "results": [
+                {
+                    "content": "[images/a.png]",
+                }
+            ],
+            "referenced_chunks": [{"chunk_id": "c1"}],
+            "decision_trace": [{"step": 1}],
+            "answer_text": "should drop",
+        }
+    )
+
+    assert response == {
+        "query": "q",
+        "evidence": [{"type": "text", "text": "t"}],
+        "evidence_text": "t",
+        "results": [
+            {
+                "content": "[images/a.png]",
+            }
+        ],
+        "referenced_chunks": [{"chunk_id": "c1"}],
+        "decision_trace": [{"step": 1}],
+    }
+
+
+def test_public_results_keep_placeholders_without_composed() -> None:
+    public = asyncio.run(
+        project_public_retrieval_response(
+            {
+                "namespace": "default",
+                "query": "q",
+                "router_used": "classic",
+                "evidence": [{"type": "text", "text": "<table>Q4</table>"}],
+                "evidence_text": "<table>Q4</table>",
+                "results": [
+                    {
+                        "chunk_id": "c1",
+                        "chunk_type": "text",
+                        "content": "见表 [tables/a.html]",
+                        "composed": [{"type": "text", "text": "见表 <table>Q4</table>"}],
+                        "score": 1,
+                        "document_id": "d1",
+                    }
+                ],
+            }
+        )
+    )
 
-    evidence_text = _render_rows_evidence(rows)
-
-    assert evidence_text.count("[E1]") == 1
-    assert evidence_text.count("[E2]") == 1
-    assert "[E3]" not in evidence_text
-    assert "[§ alpha.pdf / Alpha]" in evidence_text
-    assert "[§ beta.pdf / Beta]" in evidence_text
-    assert "[§ alpha.pdf / Alpha / One]" not in evidence_text
-    assert "[§ alpha.pdf / Alpha / Two]" not in evidence_text
-    assert "first section content" in evidence_text
-    assert "second section content" in evidence_text
-    assert "<table><tr><td>metric</td></tr></table>" in evidence_text
-    assert "[Document]" not in evidence_text
-    assert "▸" not in evidence_text
-    assert "┈" not in evidence_text
+    assert public["evidence"] == [{"type": "text", "text": "<table>Q4</table>"}]
+    assert public["results"][0]["content"] == "见表 [tables/a.html]"
+    assert "composed" not in public["results"][0]

apps/api/tests/contract/test_retrieval_document_scope.py

@@ -387,7 +387,8 @@ async def test_connected_hydration_and_assembly_reject_outside_rows(
                 )
                 assert {r["document_id"] for r in assembled} == expected
                 for row in assembled:
-                    assert "asset secret" in row["content"]
+                    # content stays raw for debug; compose keeps the placeholder intact.
+                    assert "[images/" in row["content"]
 
 
 def test_cache_scope_none_empty_and_set_identity():

apps/worker/tests/contract/test_page_memory_retrieval_contract.py

@@ -142,13 +142,15 @@ async def test_table_result_assembly_uses_summary_not_html() -> None:
 
     assert len(assembled) == 1
     content = assembled[0]["content"]
-    assert "[Table: https://assets.example.com/job-1/tables/table-1.html]" in content
-    assert "企业入驻信息登记模板" in content
-    assert "企业名称;统一社会信用代码" in content
-    assert "SHOULD NOT LEAK" not in content
-    assert "<table" not in content
-    assert "[tables/" not in content
-    assert content.index("见表") < content.index("[Table:")
+    assert content == "见表 [tables/table-1.html]"
+    composed_text = "".join(
+        str(part.get("text") or "")
+        for part in assembled[0]["composed"]
+        if part.get("type") == "text"
+    )
+    assert "<table><tr><td>SHOULD NOT LEAK</td></tr></table>" in composed_text
+    assert "[tables/" not in composed_text
+    assert composed_text.index("见表") < composed_text.index("<table")
 
 
 @pytest.mark.asyncio

deprecated/mapnav/nav/nav_compose.py

@@ -12,8 +12,6 @@
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Sequence, Set, Tuple
 
-from shared.services.retrieval.hydration.evidence_text import render_evidence_blocks
-
 from ._compat import Chunk
 from ._compat import line_node_id
 from ._compat import ToolSpace
@@ -322,14 +320,44 @@ def _render_group(
     *,
     evidence_index: int,
 ) -> str:
-    """Render one evidence block via the shared evidence renderer."""
+    """Render one evidence block as ``[E#]`` + path + bodies."""
     bodies = [_chunk_body(child.chunk) for child in selected]
-    return render_evidence_blocks(
+    return _render_evidence_blocks(
         [(group.parent_title or "", bodies)],
         start_index=evidence_index,
     )
 
 
+def _render_evidence_blocks(
+    groups: Sequence[tuple[str, Sequence[str]]],
+    *,
+    start_index: int = 1,
+) -> str:
+    parts: list[str] = []
+    index = max(1, int(start_index or 1))
+    for path, bodies in groups:
+        texts = [str(t or "").strip() for t in bodies]
+        texts = [t for t in texts if t]
+        if not texts:
+            continue
+        block: list[str] = [f"[E{index + len(parts)}]"]
+        header = str(path or "").strip()
+        if header:
+            block.append(f"[§ {header}]")
+        indent = len(texts) >= 2
+        for text in texts:
+            if indent:
+                block.append(
+                    "\n".join(
+                        ("  " + ln if ln.strip() else ln) for ln in text.splitlines()
+                    )
+                )
+            else:
+                block.append(text)
+        parts.append("\n".join(block).strip())
+    return "\n\n".join(parts)
+
+
 def _scored_flat(groups: Sequence[_ParentGroup]) -> List[Tuple[Chunk, float]]:
     out: List[Tuple[Chunk, float]] = []
     for g in groups:

packages/shared-python/shared/services/retrieval/agent_tools/tools/read.py

@@ -1,8 +1,8 @@
 """``corpus.read`` — full body content for already-located sections/chunks.
 
 Unlike ``hydration.result_assembly.assemble_retrieval_results`` (which
-down-weights ``page`` chunks to their summary — see that module's
-``_page_summary``, a deliberate trade-off for the retrieval-answer surface),
+down-weights ``page`` chunks to their summary — see ``page_summary``, a
+deliberate trade-off for the retrieval-answer surface),
 ``read`` returns the page chunk's full body content, with ``[SAME-AS <owner>
 p<N>]`` markers resolved to the owner section's text (§2 of
 ``CORPUS_SCHEMA.md``) rather than stripped or summarized. ``connect_to``

packages/shared-python/shared/services/retrieval/execution/plan.py

@@ -123,6 +123,7 @@ async def _execute_with_overrides(self, request: RetrievalQuery) -> dict[str, An
                 "namespace": request.namespace,
                 "query": request.query,
                 "router_used": "empty_query_filtered",
+                "evidence": [],
                 "evidence_text": "",
                 "answer_text": "",
                 "referenced_chunks": [],

packages/shared-python/shared/services/retrieval/execution/response_projection.py

@@ -25,6 +25,7 @@ async def project_public_retrieval_response(response: dict[str, Any]) -> dict[st
         'namespace': response.get('namespace'),
         'query': response.get('query'),
         'router_used': response.get('router_used'),
+        'evidence': response.get('evidence') or [],
         'evidence_text': response.get('evidence_text') or '',
         'answer_text': '',
         'referenced_chunks': response.get('referenced_chunks') or [],