diff --git a/.github/workflows/quickstart-tabs-check.yml b/.github/workflows/quickstart-tabs-check.yml index a3f18b54..6c680457 100644 --- a/.github/workflows/quickstart-tabs-check.yml +++ b/.github/workflows/quickstart-tabs-check.yml @@ -13,6 +13,13 @@ on: - "scripts/generate_quickstart_tabs.py" - "docs/quick-start.md" - ".github/workflows/quickstart-tabs-check.yml" +# AAASM-5529: the claim-binding gate below reads the negative controls and the +# exception the quick-start names, so an edit to either must re-run it here. +# ci.yaml already covers them under agent_assembly/**/*.py and test/**/*.py; +# these entries matter for the docs-only PR, which ci.yaml deliberately skips. + - "test/unit/test_quickstart_claim_bindings.py" + - "test/unit/test_quickstart_negative_control.py" + - "agent_assembly/exceptions/**" push: branches: - main @@ -47,3 +54,30 @@ jobs: echo "::error:: python scripts/generate_quickstart_tabs.py" exit 1 fi + + claim-bindings: + # AAASM-5529. The drift-check job above round-trips the §3 tabs as *text*: it + # proves the generated document matches the vendored snippets and nothing + # more — the snippets are never parsed, imported or executed (ruff.toml + # excludes them; they reference names they never define). + # + # This job gates a different surface: §"What just happened", where the + # quick-start states what governance did. It binds each claim to the control + # that proves it, so a claim cannot be added or reworded without someone + # naming the evidence. + # + # It runs here rather than only in ci.yaml because ci.yaml's paths allow-list + # excludes docs/**, so a docs-only PR — precisely the change that rewords a + # claim — gets no CI at all. + name: quick-start claim bindings + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v7 + + - name: Install the SDK and its dev dependencies + run: uv sync + + - name: Every documented enforcement claim names the control that proves it + run: uv run pytest test/unit/test_quickstart_claim_bindings.py -q --no-cov diff --git a/test/unit/test_quickstart_claim_bindings.py b/test/unit/test_quickstart_claim_bindings.py new file mode 100644 index 00000000..b09375ca --- /dev/null +++ b/test/unit/test_quickstart_claim_bindings.py @@ -0,0 +1,884 @@ +"""Drift gate binding the quick-start's claims to the controls that prove them. + +AAASM-5529, Epic AAASM-5526. + +Every sentence in ``docs/quick-start.md`` must be either **bound** to a control +that proves it, or **explicitly allow-listed** as making no capability claim. +There is no third state and no keyword filter. + +Why the default is inverted +--------------------------- + +Earlier revisions only scanned sentences matching an enforcement vocabulary. +Review appended three plain sentences that used none of the 21 terms — the last +of them, *"Tool bodies always execute; the policy result is recorded alongside +them"*, is the negation of the product — and all three gates stayed green. +Widening 3 → 21 terms closed the instance and not the class: **a keyword +allow-list cannot be completed, because whoever adds the claim picks the words +after reading the list.** + +So the vocabulary no longer gates anything. It survives only as a *severity +hint* in the failure message, and as the trigger for a stricter allow-list rule +(:data:`_ALLOWED` entries whose sentence matches it need a bespoke written +justification, not a category). + +There are no section-level exclusions either. An excluded section was a black +hole: the guard checked the heading still existed and said nothing about its +contents, so a claim inserted into ``## Next steps`` was never scanned at all. + +What this gate proves +--------------------- + +#. **Every sentence in the document is accounted for.** Add a sentence anywhere + — any section, any wording — and it fails until someone binds it or + allow-lists it by exact text. +#. **A binding matches a whole sentence, exactly** (``==``, never ``in``), and + exactly one binding may match a sentence. Substring containment let a + sentence carry extra unbound claims, including its own negation. +#. **Every control a binding names still exists**, extracted from the control + modules' ASTs rather than transcribed. +#. **Every claim is proven or openly unproven**, with no exempt category, and an + unproven claim must name a ticket that is *not* the ticket this module + implements — a pointer at one's own ticket resolves to a closed issue the + moment that ticket merges. +#. **Comments are stripped before scanning**, because a reader cannot see them. + Leaving them in let a bound claim be commented out of the rendered page while + the gate still counted it. + +What this gate does **not** prove +--------------------------------- + +It does not execute, import or type-check a quick-start snippet. +``quickstart_snippets/`` is a vendored verbatim copy of regions from the +``examples`` repository (``ruff.toml`` excludes it), and the snippets reference +names they never define. The ``quickstart-tabs-check`` drift job round-trips +them as *text* only. Neither job runs a snippet. + +Nor does binding a claim make it true. A binding records which control stands +behind a sentence; where none does, it says so and names the ticket. +""" + +from __future__ import annotations + +import ast +import importlib +import re +from dataclasses import dataclass, field +from pathlib import Path + +import pytest + +# NOTE: SDK symbols a claim names are resolved lazily, by module path and +# attribute name, rather than imported here. Importing them at module scope +# makes a rename a *collection* error, aborting before the assertion meant to +# catch it can run. + +#: The ticket this module implements. An unproven claim may not name it — see +#: test_an_unproven_reason_does_not_name_the_implementing_ticket. +IMPLEMENTING_TICKET = "AAASM-5529" + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_QUICK_START = _REPO_ROOT / "docs" / "quick-start.md" + +#: Modules a binding may name a control from. +_CONTROL_MODULES = ( + Path(__file__).with_name("test_quickstart_negative_control.py"), + Path(__file__).with_name("test_assembly.py"), +) + +#: NOT a gate. A severity hint in the failure message, and the trigger for the +#: stricter allow-list rule below. See the module docstring for why gating on a +#: keyword list is unsound. +_ENFORCEMENT_VOCABULARY = re.compile( + r"(?i)\bdenie[sd]\b|\bdeny\b|\bblocked\b|\bblocking\b|\bnever runs?\b" + r"|\bbefore execution\b|\bchecked against\b|\benforces?\b|\benforced\b" + r"|\bpassthrough\b|\bdiscards?\b|\bdiscarded\b|\bthrows?\b|\brejects?\b" + r"|\brouted\b|\bintercepts?\b|\binterception\b|\bgovern(s|ed|ance)?\b" + r"|\bverified\b|\bprotection\b|\bunprotected\b|\bbypass(ed|es)?\b" +) + +#: Allow-list categories. A category is only permitted for a sentence that does +#: NOT match the vocabulary above; anything that does needs a written reason. +#: The ONLY bare constant. Permitted solely for structurally non-prose lines — +#: tab labels, table rows, migration import pairs, bare link-list items — which +#: are matched by _STRUCTURAL_LINE below. Every other entry carries a written +#: justification unique to that sentence. +#: +#: The previous rule required a justification only when the sentence matched the +#: enforcement vocabulary, which is backwards: the sentences that most need +#: explaining are the ones that EVADE the vocabulary, since evading it is the +#: whole reason the scan was inverted. +_STRUCTURAL = "Structurally non-prose: a bare mkdocs tab label, which renders as a tab caption and carries no sentence." + +#: Lines that may use the bare constant. +#: +#: Fully anchored, and deliberately narrow. The previous pattern asked whether +#: a sentence STARTED with structure, not whether it was ONLY structure — so a +#: link item, a table row or a bold link was waved through on its first few +#: characters while its anchor text, which a reader sees as prose, went +#: unexamined. A payload as plain as +#: [Every tool request is permitted to proceed and its outcome captured](x.md) +#: passed. Only a bare tab label qualifies now; everything else is justified. +_STRUCTURAL_LINE = re.compile(r'^===\s+"[^"]*"$') + +#: A sentence that turns mid-way can under-claim and over-claim at once: +#: "Network-layer interception is not enabled by default, because the in-process +#: adapter already verifies every outbound request before it leaves the host." +#: The first clause is a limitation; the second is a fabrication riding along +#: under it. Rather than judge each case, the shape is rejected: an allow-listed +#: sentence may not contain a contrastive conjunction. It costs nothing on +#: genuine non-claims, because a sentence containing " but " should be split +#: regardless of what its justification says. +#: +#: Applied to EVERY entry, not only ones whose justification calls itself a +#: disclaimer. Keying off a marker phrase made the rule opt-in by the author it +#: constrains — capitalising the phrase, or omitting it, evaded the check. +#: "so" is deliberately NOT in this list. It is consequential ("therefore"), +#: not adversative, and both sentences it flagged here — "…against a local +#: policy, so you can try it with no API keys" and "…bundles the binary, so a +#: local gateway is available" — turn in the same direction they started. +#: Both attack payloads are still caught: the reviewer's used "because" and +#: the live Go case used "but". Controls for both are in the test below. +_CONTRASTIVE_CONJUNCTION = re.compile(r"(?i)\s(?:but|because|though|although|however|whereas|while)\s") + +#: "so" is handled separately, because the risk it carries is not +#: adversativeness but POLARITY CHANGE. "We don't do X but Y" is a concession; +#: "we don't do X so Y covers it" is a REASSURANCE, and reassurance is the +#: register documentation over-claims in. A negated clause followed by an +#: un-negated one is the shape that hides an affirmative capability claim +#: behind a limitation. +#: +#: Flagging "so" flat would catch six live sentences across the three repos, +#: every one of which turns the way it started. This form catches none of them +#: and still catches the payload, which is the only negative-to-positive case. +_NEGATION = re.compile(r"(?i)\b(?:not|no|never|cannot|can't|without)\b") +_SO = re.compile(r"(?i)\sso\s") + +#: A justification must be at least this long. Not a real check — no gate can +#: tell a justification from noise — but it makes reason="x" visible. +_MIN_JUSTIFICATION = 40 + + +@dataclass(frozen=True) +class ClaimBinding: + """One documented claim and the controls that stand behind it.""" + + claim_id: str + #: The claim as a WHOLE sentence, flattened. Compared with ==, not `in`. + quote: str + #: ``ClassName::test_name`` or ``test_name`` ids from _CONTROL_MODULES. + controls: tuple[str, ...] = () + #: Set when no control proves the claim. Must name a ticket, and must not + #: name IMPLEMENTING_TICKET. + unproven_reason: str = "" + #: Backticked SDK identifiers the claim names -> the module they live in. + symbols: dict[str, str] = field(default_factory=dict) + + +_DENY_CONTROLS = ( + "TestFilesystemSideEffect::test_negative_control_denied_write_leaves_no_file", + "TestNetworkSideEffect::test_negative_control_denied_egress_never_reaches_the_listener", +) +_ALLOW_AND_DENY_CONTROLS = ( + "TestFilesystemSideEffect::test_positive_control_allowed_write_creates_the_file", + *_DENY_CONTROLS, + "TestNetworkSideEffect::test_positive_control_allowed_egress_reaches_the_listener", +) + +#: AAASM-5661 measured the documented configuration: it reaches no gateway and +#: installs a deny-all fail-closed interceptor. Every control here calls +#: install_fake_core(), supplying an authoritative runtime the documented path +#: does not have, so none of them exercises what these sentences describe. +#: Binding one would launder that gap into evidence. +_DOCUMENTED_PATH_UNMEASURED = ( + "AAASM-5661: the documented configuration was measured and does not behave as this " + "sentence says. No control covers it — every control in " + "test_quickstart_negative_control.py installs a fake native core the documented " + "path does not have." +) + +BINDINGS: tuple[ClaimBinding, ...] = ( + ClaimBinding( + claim_id="tool-calls-pass-through-the-policy-gate", + quote=( + "By the end you'll have an agent — in whichever framework you already use — whose " + "tool calls pass through the Agent Assembly policy gate, and it runs **offline** " + "against a local policy, so you need no API keys and no network access to the " + "outside world." + ), + # Found by inverting the default. It states the page's central promise + # and matches no enforcement keyword, so every earlier revision of this + # gate was blind to it. + unproven_reason=_DOCUMENTED_PATH_UNMEASURED, + ), + ClaimBinding( + claim_id="governs-whichever-framework-you-use", + quote="Agent Assembly governs whichever agent framework you already use.", + # A breadth claim with no boundary. The controls prove the shared + # governed-tool chain, not "whichever framework"; per ADR 0033 §6 a + # claim like this needs a named boundary or a qualification. + unproven_reason=( + "AAASM-5768: an unbounded breadth claim. The controls prove the shared " + "governed-tool chain behind two adapter tabs, not every framework the page " + "offers, and no control enumerates them. AAASM-5768 owns bounding or " + "qualifying it." + ), + ), + ClaimBinding( + claim_id="auto-start-probes-and-starts-a-gateway", + quote=( + "Call `init_assembly()` with no `gateway_url`; the SDK probes " + "`http://localhost:7391` and, if nothing answers, runs `aasm start --mode local " + "--foreground` for you." + ), + # Measured during this ticket's scoping pass and reported: the + # [project.scripts] aasm console script shadows the bundled Rust binary + # on PATH, so find_aasm_binary() resolves the Python one, which has no + # `start` subcommand. The documented auto-start therefore cannot work + # from a clean install. + unproven_reason=( + "AAASM-5661: no control covers the documented auto-start path, and it was " + "measured not to work from a clean install — the [project.scripts] aasm " + "console script shadows the bundled binary, and the shadowing one has no " + "'start' subcommand." + ), + ), + ClaimBinding( + claim_id="no-arg-init-connects-and-appears-in-dashboard", + quote=( + "You don't configure `:50051` yourself — registration dials it automatically — so a " + "no-argument `init_assembly()` both connects and shows the agent in the dashboard." + ), + unproven_reason=_DOCUMENTED_PATH_UNMEASURED, + ), + ClaimBinding( + claim_id="gateway-returns-allow-deny-decisions", + quote=("`init_assembly()` needs to reach a **gateway** — the policy brain that returns allow/deny decisions."), + unproven_reason=_DOCUMENTED_PATH_UNMEASURED, + ), + ClaimBinding( + claim_id="init-routes-every-tool-call", + quote=( + "It registered the agent with the gateway and auto-loaded the adapter for your " + "framework — every tool call from this point on is routed through the policy gate." + ), + unproven_reason=_DOCUMENTED_PATH_UNMEASURED, + ), + ClaimBinding( + claim_id="sdk-only-enforces-on-tool-calls", + quote=( + "The in-process adapter enforces on tool calls with no network sidecar, so the " + "example runs deterministically with no real LLM or gateway round-trip." + ), + controls=_DENY_CONTROLS, + ), + ClaimBinding( + claim_id="verdict-precedes-execution", + quote=( + "The adapter intercepts the framework's tool-invocation path and asks the policy " + "engine for an allow/deny verdict before the tool actually runs." + ), + # Both halves. The negative controls prove the "before" by absence of + # the side effect; the positive controls prove the probe would have seen + # that effect had it happened. Either alone is vacuous. + controls=_ALLOW_AND_DENY_CONTROLS, + ), + ClaimBinding( + claim_id="init-wired-in-governance-label", + # Split out from the bullet it leads, once the splitter learned to keep + # closing markup with its sentence. Short, but still a claim: "wired in + # governance" asserts an outcome. + quote="**`init_assembly()` wired in governance.**", + unproven_reason=_DOCUMENTED_PATH_UNMEASURED, + ), + ClaimBinding( + claim_id="tool-calls-were-governed-label", + quote="**Tool calls were governed.**", + controls=_ALLOW_AND_DENY_CONTROLS, + ), + ClaimBinding( + claim_id="with-block-tears-everything-down", + quote=( + "**The `with` block tore everything down on exit** — adapter hooks were unwound and " + "the gateway connection closed, leaving the process exactly as it was before." + ), + # Previously exempt under the removed `kind` field. Under the inverted + # default it is a claim like any other and needs a control. + controls=("test_context_manager_shutdown_calls_adapter_unregister_hooks",), + ), + ClaimBinding( + claim_id="deny-surfaces-as-tool-execution-blocked", + quote=("If a tool call raises a `ToolExecutionBlockedError`, that is not a bug — the policy denied the call."), + controls=( + *_DENY_CONTROLS, + "TestDegradedRuntimeCannotLookProtected" + "::test_an_unavailable_native_runtime_denies_rather_than_silently_allowing", + ), + symbols={"ToolExecutionBlockedError": "agent_assembly.exceptions"}, + ), + ClaimBinding( + claim_id="sdk-only-is-the-in-process-interception-layer", + quote=( + '`mode="sdk-only"` is the in-process-only interception layer: the framework adapter ' + "enforces on tool calls, with no network sidecar to start." + ), + controls=_DENY_CONTROLS, + ), + ClaimBinding( + claim_id="other-modes-add-network-kernel-interception", + quote=( + "The other modes (`auto`, `proxy`, `ebpf`) add network/kernel interception — see " + "[Core Concepts → Modes](concepts/index.md#runtime-modes)." + ), + # This previously named AAASM-5529 — the ticket this module implements — + # which would have resolved to a closed issue the moment this merged. + # No ticket currently owns *proving* it: nothing in this SDK starts or + # probes a proxy or eBPF layer, so the honest resolution is + # qualification rather than proof, which AAASM-5766 owns. + unproven_reason=( + "AAASM-5766: no control in the Python SDK starts or probes a proxy or eBPF " + "layer, so nothing here distinguishes 'the mode adds interception' from 'the " + "mode is selected'. AAASM-5766 owns proving or qualifying it." + ), + ), +) + +#: Every sentence in the quick-start that makes no capability claim, keyed +#: exactly. A category is only permitted where the sentence does not match +#: _ENFORCEMENT_VOCABULARY; anything that does needs a written justification. +_ALLOWED: dict[str, str] = { + "Govern your first agent in about five minutes.": ( + "The page title line. An imperative naming what the reader is about to do; the substantive promise is the next sentence, which is bound." + ), + "The package is published on PyPI as [`{{ aa.python_sdk.package_name }}`]({{ aa.urls.pypi }}) (current version: `{{ aa.python_sdk.version }}`).": ( + "States where the package is distributed and at what version. Distribution, not behaviour." + ), + '=== "pip"': _STRUCTURAL, + '=== "uv"': _STRUCTURAL, + '=== "poetry"': _STRUCTURAL, + '=== "conda"': _STRUCTURAL, + "`{{ aa.python_sdk.package_name }}` is not published on conda-forge or the Anaconda default channel — create a conda environment, then install from PyPI with `pip` inside it:": ( + "A packaging-channel fact plus the workaround. Says nothing about runtime behaviour." + ), + '!!! note "`--pre` is required for now" Agent Assembly is currently published only as a pre-release on PyPI, and `pip` skips pre-releases unless you pass `--pre` (already included above).': ( + "Explains a pip flag required by the pre-release channel. Installer mechanics only." + ), + "Drop the flag once a stable (non-pre-release) version is published.": ( + "Forward-looking install instruction tied to the --pre note above it." + ), + "`{{ aa.python_sdk.package_name }}` is the pure-Python client.": ( + "Names what the base distribution contains. Packaging composition, not capability." + ), + "`{{ aa.python_sdk.package_name }}[runtime]` additionally pulls a platform wheel (`manylinux`, `macosx`) that bundles the `{{ aa.python_sdk.cli_name }}` gateway/runtime binary, so a local gateway is available without a separate install.": ( + "Names what the [runtime] extra adds to the install. Packaging composition; whether that binary is actually reachable is a separate, bound claim." + ), + "You have three options:": ("A list lead-in with no predicate of its own."), + "**Let the SDK auto-start one.**": ( + "The bold label of a bullet. The behavioural claim it introduces is the next sentence, which is bound and registered unproven." + ), + "This needs the `aasm` binary on your `PATH` (the `agent-assembly[runtime]` extra provides it).": ( + "States a prerequisite for the auto-start option. A precondition, not a claim that anything is enforced." + ), + "**Run one yourself** with `aasm start --mode local --foreground` in a separate terminal.": ( + "One of the three gateway options, given as a command to run." + ), + "For a full gateway walkthrough, see the core [Run the gateway](https://docs.agent-assembly.com/core/latest/quick-start/first-run.html) guide.": ( + "A cross-reference to the Core docs; the gateway's own claims are gated there." + ), + "**Pass an explicit URL**, as the example below does.": ( + "The third gateway option, pointing at the example below." + ), + "See [Configuration](configuration.md) for the full URL/key resolution chain (`7391` is the local default port).": ( + "A cross-reference plus the default port number." + ), + '!!! note "Local-mode transports: `:7391` REST + `:50051` gRPC" Starting local mode binds **two** loopback surfaces in one process:': ( + "Describes which ports local mode binds. Transport topology, not enforcement." + ), + "This runs the REST/dashboard API on `http://localhost:7391` (what `gateway_url` points to, and what the SDK probes and auto-starts) **and** the gRPC `AgentLifecycleService` on `127.0.0.1:50051`, which is the endpoint the native SDK uses to **register** your agent.": ( + "Maps each local-mode port to the consumer that dials it. Transport topology, not enforcement." + ), + "`:8080` is **not** the local gateway port; ignore older docs or examples that point registration there.": ( + "Corrects a wrong port number that appears in older material." + ), + "To confirm both surfaces are actually up rather than guessing from the SDK's behavior, check them directly:": ( + "Tells the reader to verify the ports themselves; the commands follow in a fenced block." + ), + "Pick your framework below — each tab is the **governance-wiring slice** (`init_assembly()` plus that framework's adapter hookup) taken verbatim from that framework's runnable example in the [examples repo](https://github.com/ai-agent-assembly/examples/tree/master/python).": ( + "Describes the provenance of the tab content. 'governance-wiring slice' names the excerpt, not an enforcement outcome." + ), + "Copy the full, runnable script — imports, tools, and the agent run — from the linked example; the slice below is the part that wires in governance.": ( + "An instruction about which lines to copy. Identifies the excerpt without claiming what the wiring achieves." + ), + 'Every example runs **offline** in `mode="sdk-only"` against a local policy, so you can try it with no API keys and no outbound network.': ( + "A claim about network usage and credentials, not about whether a denied call is stopped. The enforcement claim for this mode is bound separately." + ), + '=== "Agno"': _STRUCTURAL, + '!!! note "Version compatibility" Agno was previously published as **Phidata**; the rename replaced every `phi.*` import with `agno.*`.': ( + "Third-party framework rename note, for Agno." + ), + "Before (Phidata): `from phi.agent import Agent`": ( + "The pre-rename Agno import, shown as the before half of a two-line pair." + ), + "After (Agno): `from agno.agent import Agent`": ( + "The post-rename Agno import, shown as the after half of a two-line pair." + ), + "Source: [Agno's official Phidata → Agno migration guide](https://docs.agno.com/how-to/phidata-to-agno).": ( + "Attribution for the Agno rename note, linking that framework's own migration guide." + ), + '=== "AutoGen"': _STRUCTURAL, + "!!! note \"Version compatibility\" AutoGen's `v0.4` rewrite (2024) replaced the single `pyautogen` package's `autogen.agentchat` namespace with separate `autogen-agentchat` / `autogen-core` / `autogen-ext` packages, and `llm_config` with an explicit `model_client`.": ( + "Third-party framework migration note, for AutoGen v0.2 to v0.4." + ), + "Before (v0.2, `pyautogen`): `from autogen.agentchat import AssistantAgent`": ( + "The AutoGen v0.2 import, shown as the before half of a two-line pair." + ), + "After (v0.4+): `from autogen_agentchat.agents import AssistantAgent`": ( + "The AutoGen v0.4 import, shown as the after half of a two-line pair." + ), + "Source: [AutoGen's official v0.2 → v0.4 migration guide](https://microsoft.github.io/autogen/stable/user-guide/agentchat-user-guide/migration-guide.html).": ( + "Attribution for the AutoGen rewrite note, linking that framework's own migration guide." + ), + '=== "CrewAI"': _STRUCTURAL, + '=== "Custom (no framework)"': _STRUCTURAL, + '=== "Google ADK"': _STRUCTURAL, + '=== "Haystack"': _STRUCTURAL, + '!!! note "Version compatibility" Haystack 2.0 replaced the `farm-haystack` package with `haystack-ai` and flattened node imports into `haystack.components.*`; the two package versions cannot coexist in one environment.': ( + "Third-party framework migration note, for Haystack 1.x to 2.x." + ), + "Before (Haystack 1.x, `farm-haystack`): `from haystack.nodes import BM25Retriever`": ( + "The Haystack 1.x import, shown as the before half of a two-line pair." + ), + "After (Haystack 2.x, `haystack-ai`): `from haystack.components.retrievers.in_memory import InMemoryBM25Retriever`": ( + "The Haystack 2.x import, shown as the after half of a two-line pair." + ), + "Source: [Haystack's official migration guide](https://docs.haystack.deepset.ai/docs/migration).": ( + "Attribution for the Haystack packaging note, linking that framework's own migration guide." + ), + '=== "LangChain"': _STRUCTURAL, + '!!! note "Version compatibility" LangChain\'s import surface moved twice: `langchain-core` split out of `langchain` across the `0.1` → `0.3` series (2024), and the `1.0` rewrite (2025) moved legacy chains/agents/tools out of `langchain` entirely into `langchain-classic`.': ( + "Third-party framework migration note, for LangChain's two import moves." + ), + "Before (`<1.0`): `from langchain.agents import AgentExecutor, create_react_agent`": ( + "The pre-1.0 LangChain agents import, shown as the before half of a two-line pair." + ), + "After (`>=1.0`): `from langchain_classic.agents import AgentExecutor, create_react_agent` (requires the separate `langchain-classic` package)": ( + "The post-1.0 LangChain agents import, naming the extra package it now needs." + ), + "This SDK's own quick-start sample hit exactly this break — see AAASM-4451.": ( + "A historical note recording that this repo was affected by the LangChain break above." + ), + "Sources: [LangChain's official v1 migration guide](https://docs.langchain.com/oss/python/migrate/langchain-v1) and the [LangChain v0.3 announcement](https://www.langchain.com/blog/announcing-langchain-v0-3).": ( + "Attribution for the LangChain note, linking both upstream announcements it draws on." + ), + '=== "LangChain (Research Agent)"': _STRUCTURAL, + '=== "LangGraph"': _STRUCTURAL, + '!!! note "Version compatibility" LangGraph `1.0` deprecated `langgraph.prebuilt.create_react_agent` in favor of LangChain\'s own agent constructor.': ( + "Third-party framework deprecation note, for LangGraph 1.0." + ), + "Before (`<1.0`): `from langgraph.prebuilt import create_react_agent`": ( + "The pre-1.0 LangGraph prebuilt import, shown as the before half of a two-line pair." + ), + "After (`>=1.0`): `from langchain.agents import create_agent`": ( + "The post-1.0 replacement for the LangGraph prebuilt constructor." + ), + "Source: [LangGraph's official v1 migration guide](https://docs.langchain.com/oss/python/migrate/langgraph-v1).": ( + "Attribution for the LangGraph deprecation note, linking that project's own migration guide." + ), + '=== "LlamaIndex"': _STRUCTURAL, + '!!! note "Version compatibility" LlamaIndex `v0.10.0` (February 2024) split the monolithic `llama_index` package into a slim `llama-index-core` plus versioned per-provider packages (`llama-index-llms-openai`, etc.).': ( + "Third-party framework packaging-split note, for LlamaIndex v0.10." + ), + "An automated `llamaindex-cli upgrade` tool is provided for the migration.": ( + "Names the upstream tool that performs the LlamaIndex migration described above." + ), + "Before (`<0.10`): `from llama_index.llms import OpenAI`": ( + "The pre-0.10 LlamaIndex LLM import, shown as the before half of a two-line pair." + ), + "After (`>=0.10`): `from llama_index.llms.openai import OpenAI` (from the separate `llama-index-llms-openai` package)": ( + "The post-0.10 LlamaIndex LLM import, naming the provider package it moved into." + ), + "Source: [LlamaIndex's official v0.10 migration guide](https://www.llamaindex.ai/blog/llamaindex-v0-10-838e735948f8).": ( + "Attribution for the LlamaIndex split note, linking that project's own release post." + ), + '=== "Microsoft Agent Framework"': _STRUCTURAL, + '=== "OpenAI Agents SDK"': _STRUCTURAL, + '=== "Pydantic AI"': _STRUCTURAL, + '=== "Semantic Kernel"': _STRUCTURAL, + '=== "smolagents"': _STRUCTURAL, + '!!! note "Version compatibility" smolagents `v1.14.0` (April 2025) renamed `HfApiModel` to `InferenceClientModel` to reflect that it wraps any Hugging Face Inference Provider, not just the HF Hub; backward-compatible re-export was restored in `v1.24.0`.': ( + "Third-party framework rename note, for smolagents v1.14 and its v1.24 re-export." + ), + "Before (`<1.14`): `from smolagents import HfApiModel`": ( + "The pre-1.14 smolagents model import, shown as the before half of a two-line pair." + ), + "After (`>=1.14`): `from smolagents import InferenceClientModel`": ( + "The post-1.14 smolagents model import, after the class was renamed." + ), + "Source: [smolagents releases](https://github.com/huggingface/smolagents/releases).": ( + "Attribution for the smolagents rename note, linking that project's release list." + ), + '=== "Strands Agents"': _STRUCTURAL, + '**`mode="sdk-only"` kept it offline.**': ( + "The bold label of a bullet, claiming only that no network was used. The enforcement half of the bullet is the next sentence, which is bound to the deny controls." + ), + "That's the product working.": ( + "A one-clause reassurance attached to the ToolExecutionBlockedError sentence before it, which is bound." + ), + "See [Handling allow/deny decisions](guides/handling-decisions.md) for how to catch and respond to those, and [Troubleshooting](troubleshooting.md) if `init_assembly()` itself raised.": ( + "A cross-reference. It reads like a claim only through the linked page's title; the claims live on that page." + ), + "It's the most portable mode and the best choice for deterministic, offline examples and tests.": ( + "A recommendation about which mode to pick for examples. A preference, not a capability." + ), + "**[Core Concepts](concepts/index.md)** — the adapter pattern, the `init_assembly()` lifecycle, and the modes/enforcement model.": ( + "A Next-steps link item. Its anchor text lists topics covered elsewhere; the claims live on the Core Concepts page and are gated there." + ), + "**[Examples](examples/index.md)** — wire the SDK into the framework you actually use.": ( + "A Next-steps link item pointing at the examples index. An invitation to read further, asserting nothing about enforcement." + ), + "**[Configuration](configuration.md)** — drop the hard-coded URL and key; let the resolver chain find them.": ( + "A Next-steps link item about configuration ergonomics: where the URL and key come from, not what governance does." + ), +} + + +_FRONT_MATTER = re.compile(r"\A---\n.*?\n---\n", re.DOTALL) +_FENCE = re.compile(r"```.*?```", re.DOTALL) +#: Comments are invisible to a reader, so a bound claim commented out of the +#: rendered page must not still satisfy this gate. Stripped for the same reason +#: fences are. Go's CommonMark HTML blocks and MDX's {/* */} form are covered +#: too, so the three gates strip the same things. +_HTML_COMMENT = re.compile(r"", re.DOTALL) +_MDX_COMMENT = re.compile(r"\{/\*.*?\*/\}", re.DOTALL) +_LIST_MARKER = re.compile(r"(?m)^\s*(?:[-*+]|\d+\.)\s+") +_UNIT_SPLIT = re.compile(r"(?m)^(?=\s*(?:[-*+]|\d+\.)\s|\|)") +#: '.' and '?' only. '!' is not a terminator here because mkdocs admonitions +#: open with '!!! note', which would otherwise split into a bare '!!!' unit. +#: +#: Closing markup between the terminator and the space is consumed WITH the +#: sentence, so "**Tool calls were governed.**" is its own unit instead of +#: running into the sentence after it. Without that, a bold lead-in label and +#: the claim it introduces were one string, and binding the pair covered both. +#: A backtick is deliberately NOT in the trailing class: inline code such as +#: `phi.*` would otherwise be read as a sentence end and split mid-sentence. +_SENTENCE_END = re.compile(r"[.?][*)\]_\"']*(?=\s)") + + +def _split_sentences(unit: str) -> list[str]: + """Split on terminators, keeping the terminator and its markup attached.""" + out: list[str] = [] + start = 0 + for match in _SENTENCE_END.finditer(unit): + out.append(unit[start : match.end()]) + start = match.end() + out.append(unit[start:]) + return out + + +def _document() -> str: + """Read the quick-start with line endings normalised to LF. + + Without this the paragraph split never fires on a CRLF checkout and the + whole section collapses into one sentence. Node's Windows CI legs caught + exactly that; this repo's CI is Linux-only, so it is latent here. + """ + return _QUICK_START.read_text(encoding="utf-8").replace("\r\n", "\n") + + +def _scanned_occurrences() -> list[tuple[str, str]]: + """Return every ``(sentence, section)`` OCCURRENCE in the whole document. + + A list, not a dict. Keying by sentence collapsed duplicates before anything + counted them, so ``matched == 1`` could only ever be 0 or 1 and a bound true + sentence could be pasted into a section that inverts its meaning — an + observe-mode block, a "what not to do" block — and still count once. Section + attribution was last-write-wins for the same reason. + + No section is skipped. A section-level exclusion was a black hole: the guard + checked the heading still existed and said nothing about its contents, so a + claim inserted into an excluded section was never scanned. + """ + body = _document() + body = _FRONT_MATTER.sub("", body) + for pattern in (_FENCE, _HTML_COMMENT, _MDX_COMMENT): + body = pattern.sub("\n\n", body) + + occurrences: list[tuple[str, str]] = [] + section = "(preamble)" + for chunk in re.split(r"(?m)^(#{1,6} .*)$", body): + if chunk is None: + continue + if re.match(r"^#{1,6} ", chunk): + section = chunk.strip() + continue + for paragraph in chunk.split("\n\n"): + for unit in _UNIT_SPLIT.split(paragraph): + for raw in _split_sentences(_LIST_MARKER.sub("", unit)): + flat = re.sub(r"\s+", " ", raw).strip() + if flat: + occurrences.append((flat, section)) + return occurrences + + +def _control_node_ids() -> set[str]: + """Extract control ids from the control modules' ASTs, not transcribed.""" + node_ids: set[str] = set() + for module in _CONTROL_MODULES: + tree = ast.parse(module.read_text(encoding="utf-8")) + for node in tree.body: + if isinstance(node, ast.ClassDef): + for child in node.body: + if isinstance(child, ast.FunctionDef | ast.AsyncFunctionDef) and child.name.startswith("test_"): + node_ids.add(f"{node.name}::{child.name}") + elif isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) and node.name.startswith("test_"): + node_ids.add(node.name) + return node_ids + + +class TestTheGateCanSeeWhatItGates: + """Positive controls. An empty parse and a clean result look identical.""" + + def test_the_whole_document_is_read_and_split(self) -> None: + assert len(_scanned_occurrences()) > 60, "too few sentences parsed from the whole quick-start" + + def test_the_scan_covers_every_section_including_the_last(self) -> None: + sections = {section for _, section in _scanned_occurrences()} + assert len(sections) >= 6, f"the scan reached only {len(sections)} sections: {sections}" + assert "## Next steps" in sections, ( + "'## Next steps' is not in the scan. It used to be excluded by name, which made it " + "a black hole: a claim inserted there was never seen. It must be scanned." + ) + + def test_comments_are_stripped_before_scanning(self) -> None: + """A commented-out sentence must not satisfy a binding. + + Positive control for the strip: the document contains HTML comments, and + none of their content may appear in the scan. + """ + assert "