test(scope): differential regression suite proves agent-scope-hardening holds (Spec 105 PR H1)

.github/workflows/scope-latency.yml

@@ -0,0 +1,130 @@
+name: Scope Latency (Spec 105 FR-011)
+
+# T112a: the FR-011 merge-base regression gate. internal/server's
+# scope_latency_test.go measures administrator p95 for four operations
+# (retrieve_tools, read_cache, prompts/list, tools/list) on the 527-tool
+# LiveMCPBench snapshot; this workflow runs that SAME test file — HEAD's
+# copy, unmodified — against both the merge-base revision and HEAD, then
+# fails if administrator p95 regressed by more than max(10%, 5ms) on any
+# operation, or if either run produced no measurement for one.
+#
+# Why HEAD's test file runs against the merge-base checkout, not the
+# merge-base's own copy: the test file is test-only and backward compatible
+# with the pre-feature production API (it calls handleRetrieveTools,
+# handleReadCache, server.HandleMessage and directServer.HandleMessage,
+# none of which this feature's PRs changed the signature of), so copying it
+# over the merge-base checkout compiles and runs there unchanged — this is
+# the ONLY way to compare "the same measurement code" across two revisions
+# whose production code differs (research D10).
+#
+# Non-race, single job, reference (GitHub-hosted ubuntu-latest) runner: the
+# race detector's instrumentation overhead swamps the microsecond/millisecond
+# deltas this measures (scope_latency_test.go's raceEnabled skip already
+# documents this), and a self-hosted "reference runner" is not available to
+# this repository, so ubuntu-latest is the closest available reference
+# baseline, run non-race for the same reason the in-repo tests are.
+#
+# A generous, well-reasoned threshold (10% relative or 5ms absolute, per
+# FR-011) is used deliberately over a tighter one: this epic already hit real
+# CI flakiness from over-tight allocation/timing assertions on PRs F and G
+# today (2026-09-20) — see CLAUDE.md's cross-model-review note on that PR's
+# history. This workflow is NOT (yet) a required status check; a tight gate
+# that flakes is worse than no gate, so start generous and tighten only with
+# real multi-run evidence.
+
+on:
+  pull_request:
+    paths:
+      - "internal/server/**"
+      - "internal/index/**"
+      - "cmd/scope-latency-compare/**"
+      - ".github/workflows/scope-latency.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  scope-latency:
+    name: FR-011 merge-base regression
+    runs-on: ubuntu-latest
+    # Advisory only: a flaky or newly-added perf gate must not block merges
+    # before it has a track record. Flip to false once the maintainer has
+    # watched it green on a handful of real PRs.
+    continue-on-error: true
+    steps:
+      - name: Checkout HEAD
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          path: head
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version: "1.26"
+          cache: true
+          cache-dependency-path: head/go.sum
+
+      - name: Determine merge-base
+        id: mergebase
+        working-directory: head
+        run: |
+          git fetch origin "${{ github.event.pull_request.base.sha }}" --depth=1
+          base_sha=$(git merge-base "${{ github.event.pull_request.base.sha }}" HEAD)
+          echo "sha=$base_sha" >> "$GITHUB_OUTPUT"
+          echo "Merge-base: $base_sha"
+
+      - name: Checkout merge-base into a second directory
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          path: base
+          ref: ${{ steps.mergebase.outputs.sha }}
+
+      - name: Copy HEAD's latency test + fixtures over the merge-base checkout
+        run: |
+          set -euo pipefail
+          cp head/internal/server/scope_latency_test.go base/internal/server/scope_latency_test.go
+          rm -rf base/internal/server/testdata/scope_latency
+          if [ -d head/internal/server/testdata/scope_latency ]; then
+            cp -R head/internal/server/testdata/scope_latency base/internal/server/testdata/scope_latency
+          fi
+          # scope_fixture_test.go, mcp_retrieve_scope_test.go (loadDeferredLargeCorpus,
+          # createTestProxyWithRuntime) and friends are shared test helpers the
+          # latency file calls into; the merge-base checkout already has its OWN
+          # copies of those (they predate this PR on every commit in the merge
+          # order — Phase 1's T001/T002 landed with PR A, long before H1), so
+          # nothing else needs copying across. A future PR that adds a NEW
+          # shared helper the latency file depends on must add it here too.
+
+      - name: Run scope latency measurements (merge-base)
+        working-directory: base
+        env:
+          SCOPE_LATENCY_RESULTS_FILE: ${{ github.workspace }}/base-results.txt
+        run: |
+          go test ./internal/server/... \
+            -run 'TestRetrieveTools_ScopeLatency_ScopedVsAdmin|TestScopeLatency_' \
+            -v -timeout 10m
+
+      - name: Run scope latency measurements (HEAD)
+        working-directory: head
+        env:
+          SCOPE_LATENCY_RESULTS_FILE: ${{ github.workspace }}/head-results.txt
+        run: |
+          go test ./internal/server/... \
+            -run 'TestRetrieveTools_ScopeLatency_ScopedVsAdmin|TestScopeLatency_' \
+            -v -timeout 10m
+
+      - name: Compare administrator p95 across revisions
+        working-directory: head
+        run: |
+          go run ./cmd/scope-latency-compare "${{ github.workspace }}/base-results.txt" "${{ github.workspace }}/head-results.txt"
+
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: scope-latency-results
+          path: |
+            base-results.txt
+            head-results.txt
+          retention-days: 14

ROADMAP.md

@@ -242,8 +242,8 @@ graph LR
 
   classDef done fill:#1f7a1f,stroke:#0d3d0d,color:#ffffff;
   classDef todo fill:#6e7781,stroke:#3d4248,color:#ffffff;
-  class scope_fix_target_tier,scope_fix_tail_log,scope_fix_set_profile,scope_fix_read_cache,scope_fix_prompts_profile_url,scope_retrieve_tools,scope_direct_publication,scope_selectable_profile_predicate,scope_cache_legacy_invalidation,scope_log_attribution,scope_target_identity_producers,scope_fix_stored_script_admin done;
-  class scope_refusal_shapes,scope_regression_suite todo;
+  class scope_fix_target_tier,scope_fix_tail_log,scope_fix_set_profile,scope_fix_read_cache,scope_fix_prompts_profile_url,scope_retrieve_tools,scope_direct_publication,scope_refusal_shapes,scope_selectable_profile_predicate,scope_cache_legacy_invalidation,scope_log_attribution,scope_target_identity_producers,scope_fix_stored_script_admin done;
+  class scope_regression_suite todo;
 ```
 
 | Task | Status | Refs |
@@ -255,7 +255,7 @@ graph LR
 | FR-006 + FR-004 (deleted pin): aggregated prompts authorized by canonical registration owner; profile URL / set_profile stop enumerating on a deleted pin | 🟢 Done | #1227 |
 | FR-005: retrieve_tools filters by scope BEFORE limiting; indexed counts, usage ranking, debug output and session risk computed over the authorized population only | 🟢 Done | #1325 |
 | FR-008: direct-surface definitions take owner and tier from their own registration identity at every publication seam, both skew directions, full and deferred | 🟢 Done | #1326 |
-| FR-010: scope-first refusal precedence; dispatch denials and 'available servers' never name hidden servers; describe_tool not-found and alias resolution computed over the authorized corpus | ⚪ Todo | — |
+| FR-010: scope-first refusal precedence; dispatch denials and 'available servers' never name hidden servers; describe_tool not-found and alias resolution computed over the authorized corpus | 🟢 Done | #1328 |
 | FR-003/FR-004 remainder: selectable-profile predicate for UNPINNED tokens on /mcp/p/<slug>, /mcp/p, /mcp/p/ and set_profile; identical status+body across missing / deleted / not-selectable / pin-mismatch / no-profiles (#1225 + #1227 follow-up lists) | 🟢 Done | #1283 |
 | FR-002 + FR-001 remainder: legacy/unstamped and internal (registry, guesser) cache entries refused for every caller and durably invalidated; monotone recursive provenance; existence-non-disclosing refusal on MCP and REST (#1226 follow-up list) | 🟢 Done | #1282 |
 | FR-007 remainder: per-record canonical log ownership (a/b vs a_b share one file), filter-before-limit + authorized lines_returned, subject-bound OAuth-callback logging, canonical container ownership in Docker cleanup (#1224 follow-up list) | 🟢 Done | #1284 |
@@ -888,7 +888,7 @@ graph LR
 | Web UI + macOS app UX audit | In progress | P0 | — |  |  |
 | Release qualification gate (auto-QA matrix blocks the tag) | In progress | P0 | — | [081-release-qa-gate](./specs/081-release-qa-gate/) |  |
 | Action log / transparency — info at a glance | In progress | P1 | — |  |  |
-| Agent-token scope hardening: every MCP request authorized by its own scope (spec 105) | In progress | P1 | 82/112 (73%) | [105-agent-scope-hardening](./specs/105-agent-scope-hardening/) |  |
+| Agent-token scope hardening: every MCP request authorized by its own scope (spec 105) | In progress | P1 | 94/113 (83%) | [105-agent-scope-hardening](./specs/105-agent-scope-hardening/) |  |
 | Token-efficiency benchmark: measured savings, published results | In progress | P1 | 62/64 (97%) | [103-token-bench](./specs/103-token-bench/) |  |
 | Telemetry identity & data quality (machine_id + CI-filter hardening) | In progress | P1 | — |  |  |
 | Telemetry v7: honest funnel + churn instrumentation | In progress | P1 | — | [080-telemetry-v7-churn](./specs/080-telemetry-v7-churn/) |  |
@@ -1034,6 +1034,6 @@ Legend: `shipped` ≥95% checked · `in-flight` 1–94% · `drafted` 0% · `—`
 | [102-schema-deferred](./specs/102-schema-deferred/) | `shipped` | 89/89 (100%) |
 | [103-token-bench](./specs/103-token-bench/) | `shipped` | 62/64 (97%) |
 | [104-auto-routing-mode](./specs/104-auto-routing-mode/) | — | — |
-| [105-agent-scope-hardening](./specs/105-agent-scope-hardening/) | `in-flight` | 82/112 (73%) |
+| [105-agent-scope-hardening](./specs/105-agent-scope-hardening/) | `in-flight` | 94/113 (83%) |
 | [106-security-residual-fixes](./specs/106-security-residual-fixes/) | `shipped` | 18/19 (95%) |
 | [107-server-edition-sso-hardening](./specs/107-server-edition-sso-hardening/) | `shipped` | 126/126 (100%) |

cmd/scope-latency-compare/main.go

@@ -0,0 +1,137 @@
+// Command scope-latency-compare is the T112a merge-base gate for Spec 105
+// FR-011: it compares the administrator p95 latency for each of the four
+// scope-hardening operations (retrieve_tools, read_cache, prompts/list,
+// tools/list) between two revisions and fails if HEAD regressed by more than
+// max(10%, 5ms) on any operation, or if either side is missing a
+// measurement for any operation.
+//
+// Input: two files, each produced by internal/server's scope_latency_test.go
+// (via SCOPE_LATENCY_RESULTS_FILE) as newline-separated "operation=nanoseconds"
+// pairs. No third-party dependency — stdlib only, per research D10.
+//
+// Usage: scope-latency-compare <merge-base-results-file> <head-results-file>
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// requiredOperations is FR-011's fixed operation list; every result file
+// MUST report a measurement for every one of these, or the comparison fails
+// closed (T112a: "fails if either side yields no measurement for any
+// operation").
+var requiredOperations = []string{"retrieve_tools", "read_cache", "prompts/list", "tools/list"}
+
+const minAbsoluteBudget = 5 * time.Millisecond
+const relativeBudget = 0.10 // 10%
+
+func main() {
+	if len(os.Args) != 3 {
+		fmt.Fprintln(os.Stderr, "usage: scope-latency-compare <merge-base-results-file> <head-results-file>")
+		os.Exit(2)
+	}
+	baseFile, headFile := os.Args[1], os.Args[2]
+
+	base, err := readResults(baseFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "reading merge-base results %s: %v\n", baseFile, err)
+		os.Exit(1)
+	}
+	head, err := readResults(headFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "reading HEAD results %s: %v\n", headFile, err)
+		os.Exit(1)
+	}
+
+	lines, failures := compare(base, head, baseFile, headFile)
+	for _, line := range lines {
+		fmt.Println(line)
+	}
+	if len(failures) > 0 {
+		fmt.Fprintln(os.Stderr, "\nscope-latency-compare: FAIL")
+		for _, f := range failures {
+			fmt.Fprintln(os.Stderr, "  - "+f)
+		}
+		os.Exit(1)
+	}
+	fmt.Println("\nscope-latency-compare: PASS — every operation within FR-011's merge-base regression budget")
+}
+
+// compare evaluates every required operation and returns (a) one
+// human-readable report line per operation that had a measurement on both
+// sides, and (b) the list of failure reasons — missing measurements or a
+// regression exceeding max(10%, 5ms). baseFile/headFile are used only to
+// name the missing side in a failure message.
+func compare(base, head map[string]int64, baseFile, headFile string) (lines, failures []string) {
+	for _, op := range requiredOperations {
+		baseNS, baseOK := base[op]
+		headNS, headOK := head[op]
+		if !baseOK {
+			failures = append(failures, fmt.Sprintf("%s: no measurement on the merge-base side (%s)", op, baseFile))
+			continue
+		}
+		if !headOK {
+			failures = append(failures, fmt.Sprintf("%s: no measurement on the HEAD side (%s)", op, headFile))
+			continue
+		}
+
+		baseDur := time.Duration(baseNS)
+		headDur := time.Duration(headNS)
+		budget := time.Duration(float64(baseDur) * relativeBudget)
+		if budget < minAbsoluteBudget {
+			budget = minAbsoluteBudget
+		}
+		regression := headDur - baseDur
+		status := "OK"
+		if regression > budget {
+			status = "REGRESSION"
+			failures = append(failures, fmt.Sprintf(
+				"%s: administrator p95 regressed by %s (base=%s head=%s), exceeding the max(10%%, 5ms) budget of %s",
+				op, regression, baseDur, headDur, budget))
+		}
+		lines = append(lines, fmt.Sprintf("%-16s base=%-12s head=%-12s delta=%-12s budget=%-10s %s",
+			op, baseDur, headDur, regression, budget, status))
+	}
+	return lines, failures
+}
+
+// readResults parses a scope_latency_test.go results file: one
+// "operation=nanoseconds" pair per line, blank lines ignored. Multiple lines
+// for the same operation (a test rerun, or -count>1) keep the LAST value.
+func readResults(path string) (map[string]int64, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	out := make(map[string]int64)
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" {
+			continue
+		}
+		parts := strings.SplitN(line, "=", 2)
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("malformed line %q (want operation=nanoseconds)", line)
+		}
+		ns, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("malformed duration in line %q: %w", line, err)
+		}
+		out[strings.TrimSpace(parts[0])] = ns
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	if len(out) == 0 {
+		return nil, fmt.Errorf("no measurements found in %s", path)
+	}
+	return out, nil
+}