fix(eval): enforce deciding evidence in knowledge promotion

.github/workflows/ci.yml

@@ -43,11 +43,6 @@ jobs:
       - name: Verify packed package lifecycle
         run: pnpm verify:package
 
-      - name: Verify packed package with Eval 0.174
-        run: node scripts/verify-package.mjs
-        env:
-          AGENT_KNOWLEDGE_EVAL_VERSION: 0.174.0
-
   official-optimizers:
     runs-on: ubuntu-latest
     timeout-minutes: 20

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 17.0.0 — 2026-09-14
+
+Requires Eval `>=0.181.0 <0.182.0` and uses its explicit held-out release decision type.
+Memory activation checks the deciding safety interval and refuses insufficient or indeterminate evidence.
+RAG promotion also requires Eval's complete final decision; a positive diagnostic bound cannot override an inconclusive result.
+Source-unit mappings are captured before execution and bound to resume identity; repetitions retain their measured coverage.
+Retrieval, RAG, and KB-policy optimization inherit Eval's optional source-unit claims and durable final-evidence controls.
+Long finite run lineages remain readable; callers can set `maxAncestors` when they need a bound.
+
 ## 16.0.1 — 2026-09-12
 
 Run-scoped promotion verifies and freezes source pages, including cited support, before writing the shared store.

README.md

@@ -9,7 +9,7 @@ Supply application callbacks for those decisions, or use `@tangle-network/agent-
 ## Install
 
 ```bash
-pnpm add @tangle-network/agent-knowledge@15.0.3 @tangle-network/agent-eval@0.180.0 @tangle-network/agent-interface@2.6.0
+pnpm add @tangle-network/agent-knowledge@17.0.0 @tangle-network/agent-eval@0.181.0 @tangle-network/agent-interface@2.6.0
 ```
 
 Requires Node.js 20.19 or later.
@@ -411,13 +411,29 @@ Official external methods must report observed package identity.
 Custom in-process methods have no external package identity, so their behavior must be covered by `executionRef`.
 Treat `accountingComplete: false` as incomplete evidence for activation.
 
+Retrieval, RAG, serialized-candidate, and KB-policy optimization accept Eval's optional `claim` and `finalEvidence` options.
+Use `claim.independentUnit` to group questions from the same source.
+A `new-units` claim requires separate source units for development and final evaluation.
+Results retain scenario scores, source-unit scores, and both observation counts.
+Claim metadata declares the intended scope; it does not authenticate labels or establish certification.
+
+For fresh final evidence, pass a shared durable ledger from `openFinalEvidenceLedger()` in `@tangle-network/agent-eval/experiment`.
+The policy also requires a request ID and the actual evaluator's content digest.
+Eval reserves final units before search and records exposure before measurement.
+Interrupted measurements consume that evidence, and another request cannot restore its freshness.
+See [Eval's integrity guide](https://github.com/tangle-network/agent-eval/blob/main/docs/evaluation-integrity.md) for the full controls and limits.
+
 Retrieval and answer generation remain callbacks.
 This lets the same evaluation code work with local search, vector databases, hybrid search, rerankers, and hosted RAG services.
 Adaptive diagnosis, acquisition, and update callbacks finish before retrieval or RAG final scoring starts.
 Only answer evaluation, the terminal promotion decision, and the returned result can observe selected configurations.
 Answer-quality evidence must name at least two final scenario IDs, immutable dataset and evaluator references, non-empty finite metrics, and observed cost accounting.
 Promotion also requires `answerQualityCostCeiling`.
 
+`calibrateRagAnswerJudge()` checks supplied strong and weak fixtures; it does not measure an evaluator's error rates.
+For evaluator admission, use `auditEvaluator()` from `@tangle-network/agent-eval/meta-eval` with actual judgments of independently verified controls.
+The application must enforce evaluator and auditor separation and retain evidence for the labels.
+
 ## Integrate memory systems
 
 `@tangle-network/agent-knowledge/memory` defines `AgentMemoryAdapter` and adapters for Mem0, Graphiti, and Neo4j Agent Memory.
@@ -429,6 +445,12 @@ Use them to compare a provider against no memory or another provider on the same
 `runAgentMemoryImprovement` accepts a complete `OptimizationMethod`, evaluates each serialized configuration in an isolated provider branch, and activates only a winner that passes a separate final comparison.
 Set `implementationRef` to `git:<40 lowercase hex>` or `sha256:<64 lowercase hex>` covering the installed implementation, method configuration, candidate construction, execution behavior, and external configuration so incompatible state cannot resume.
 The run records one immutable candidate reference for each memory configuration and refuses cached results if that reference changes.
+Critical dimensions use Eval's deciding interval and observation minimum.
+Missing, insufficient, or indeterminate safety evidence holds activation.
+An interval that crosses the safety margin remains uncertain; it is not reported as an observed regression.
+Set `significance.independentUnitByScenarioId` when final sequences share a source unit.
+The run captures this mapping before execution and binds it to resume identity.
+Paired repetitions retain their coverage without increasing the independent-unit count.
 Each improvement candidate declares a maximum for one sequence and one recovery attempt.
 The adapter must enforce that maximum with its provider before starting external work.
 The adapter callback must call `recordExternalCost()` with each observed charge.

api-surface.json

@@ -296,7 +296,7 @@
       "KnowledgeRelationWalkOptions": "value fe4fbf78ea49",
       "KnowledgeRelationWalkStep": "value dce4639198b3",
       "KnowledgeRelease": "value 28df725e17fa",
-      "KnowledgeReleaseInput": "value 113e4cc7bdc7",
+      "KnowledgeReleaseInput": "value 844c1ad8d42f",
       "KnowledgeReleaseReport": "value 5f72778544bd",
       "KnowledgeResearchLoopContext": "value 1a4d66267646",
       "KnowledgeResearchLoopDecision": "value 648813512b86",

package.json

@@ -84,14 +84,14 @@
     "zod": "4.5.4"
   },
   "peerDependencies": {
-    "@tangle-network/agent-eval": ">=0.174.0 <0.181.0",
+    "@tangle-network/agent-eval": ">=0.181.0 <0.182.0",
     "@tangle-network/agent-interface": "^2.0.0"
   },
   "devDependencies": {
     "@arethetypeswrong/cli": "^0.18.5",
     "@biomejs/biome": "^2.5.11",
     "@neo4j-labs/agent-memory": "0.4.1",
-    "@tangle-network/agent-eval": "0.180.0",
+    "@tangle-network/agent-eval": "0.181.0",
     "@tangle-network/agent-interface": "2.0.0",
     "@types/node": "^26.4.0",
     "mem0ai": "3.1.7",

scripts/verify-official-optimizers.mjs

@@ -18,7 +18,7 @@ const agentEvalVersion = sourcePackage.devDependencies?.['@tangle-network/agent-
 if (!/^\d+\.\d+\.\d+$/.test(agentEvalVersion)) {
   throw new Error('@tangle-network/agent-eval must have one exact development pin')
 }
-const expectedEvalPeerRange = '>=0.174.0 <0.181.0'
+const expectedEvalPeerRange = expectedPeerRange(agentEvalVersion)
 if (sourcePackage.peerDependencies?.['@tangle-network/agent-eval'] !== expectedEvalPeerRange) {
   throw new Error(
     `@tangle-network/agent-eval peer range must be ${expectedEvalPeerRange} to match the development pin`,

scripts/verify-package.mjs

@@ -67,12 +67,12 @@ const agentCorePackage = '@tangle-network/agent-core'
 const agentInterfacePackage = '@tangle-network/agent-interface'
 const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..')
 const sourcePackage = JSON.parse(readFileSync(join(repoRoot, 'package.json'), 'utf8'))
-const agentEvalVersion =
-  process.env.AGENT_KNOWLEDGE_EVAL_VERSION ?? exactDevelopmentPin(sourcePackage, agentEvalPackage)
-if (!['0.174.0', '0.175.0', '0.176.0', '0.177.0', '0.178.0', '0.179.0', '0.180.0'].includes(agentEvalVersion)) {
+const agentEvalDevelopmentVersion = exactDevelopmentPin(sourcePackage, agentEvalPackage)
+const agentEvalVersion = process.env.AGENT_KNOWLEDGE_EVAL_VERSION ?? agentEvalDevelopmentVersion
+if (!caretAdmits(`^${agentEvalDevelopmentVersion}`, agentEvalVersion)) {
   throw new Error(`unsupported Eval compatibility test version: ${agentEvalVersion}`)
 }
-const agentEvalPeerRange = '>=0.174.0 <0.181.0'
+const agentEvalPeerRange = expectedPeerRange(agentEvalDevelopmentVersion)
 const agentInterfaceVersion = exactDevelopmentPin(sourcePackage, agentInterfacePackage)
 const agentInterfacePeerRange = expectedPeerRange(agentInterfaceVersion)
 const zodVersion = exactVersion(sourcePackage.dependencies?.zod, 'zod runtime dependency')

src/memory/improvement/promotion.ts

@@ -41,26 +41,30 @@ export function decidePromotion<TConfig extends JsonValue>(input: {
     options.significance,
   )
   const tolerance = options.criticalDimensionTolerance ?? 0.05
-  const criticalDimensions = (options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS).map(
+  const criticalChecks = (options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS).map(
     (dimension) => {
       const expectedN =
         applicableSequenceCount(options.finalSequences, dimension) * (options.reps ?? 1)
       const pairs = pairedDimension(finalEvaluation, dimension)
       const comparison = heldoutSignificance(pairs, {
         ...options.significance,
-        deltaThreshold: 0,
+        deltaThreshold: -tolerance,
       })
+      const measured = expectedN > 0 && pairs.before.length === expectedN
       return {
-        dimension,
-        n: comparison.n,
-        expectedN,
-        measured: expectedN > 0 && comparison.n === expectedN,
-        meanDelta: comparison.bootstrap.mean,
-        low: comparison.bootstrap.low,
-        high: comparison.bootstrap.high,
-        tolerance,
-        regressed:
-          expectedN > 0 && comparison.n === expectedN && comparison.bootstrap.low < -tolerance,
+        decision: comparison.decision,
+        report: {
+          dimension,
+          n: comparison.n,
+          expectedN,
+          measured,
+          meanDelta: comparison.bootstrap.mean,
+          low: comparison.decision.low,
+          high: comparison.decision.high,
+          tolerance,
+          regressed:
+            measured && !comparison.decision.indeterminate && comparison.decision.high < -tolerance,
+        },
       }
     },
   )
@@ -84,22 +88,32 @@ export function decidePromotion<TConfig extends JsonValue>(input: {
   if (!significance.significant) {
     reasons.push(
       significance.fewRuns
-        ? `only ${significance.n} paired final cells; more are required`
+        ? `only ${significance.n} paired final observations; at least ${significance.minimumRequired} are required`
         : 'final lift is not confidently above the promotion threshold',
     )
   }
   if (winnerScore < (options.minFinalScore ?? 0)) {
     reasons.push(`winner final score ${winnerScore} is below the required minimum`)
   }
-  for (const dimension of criticalDimensions) {
+  for (const { report: dimension, decision } of criticalChecks) {
     if (!dimension.measured) {
       reasons.push(
         dimension.expectedN === 0
           ? `critical dimension ${dimension.dimension} has no applicable final histories`
           : `critical dimension ${dimension.dimension} was measured on ${dimension.n}/${dimension.expectedN} applicable paired final cells`,
       )
+    } else if (!decision.sufficient) {
+      reasons.push(
+        `critical dimension ${dimension.dimension} needs at least ${decision.minimumPairs} paired final observations; measured ${decision.n}`,
+      )
+    } else if (decision.indeterminate) {
+      reasons.push(`critical dimension ${dimension.dimension} has an indeterminate final interval`)
     } else if (dimension.regressed) {
-      reasons.push(`${dimension.dimension} may regress beyond ${tolerance}`)
+      reasons.push(`critical dimension ${dimension.dimension} regresses beyond ${tolerance}`)
+    } else if (!decision.promote) {
+      reasons.push(
+        `critical dimension ${dimension.dimension} does not exclude a regression beyond ${tolerance}`,
+      )
     }
   }
   return {
@@ -109,7 +123,7 @@ export function decidePromotion<TConfig extends JsonValue>(input: {
     winnerScore,
     lift: winnerScore - baselineScore,
     significance,
-    criticalDimensions,
+    criticalDimensions: criticalChecks.map((check) => check.report),
   }
 }
 
@@ -124,6 +138,13 @@ export function normalizedPromotionPolicy<TConfig extends JsonValue>(
       resamples: options.significance?.resamples ?? 2000,
       seed: options.significance?.seed ?? 1337,
       statistic: options.significance?.statistic ?? 'mean',
+      ...(options.significance?.independentUnitByScenarioId === undefined
+        ? {}
+        : {
+            independentUnitByScenarioId: [...options.significance.independentUnitByScenarioId].sort(
+              ([left], [right]) => (left < right ? -1 : left > right ? 1 : 0),
+            ),
+          }),
     },
     criticalDimensions: [...(options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS)],
     criticalDimensionTolerance: options.criticalDimensionTolerance ?? 0.05,

src/memory/improvement/run.ts

@@ -47,6 +47,21 @@ import { assertMemoryImprovementOptions } from './validation'
 export async function runAgentMemoryImprovement<TConfig extends JsonValue>(
   options: RunAgentMemoryImprovementOptions<TConfig>,
 ): Promise<RunAgentMemoryImprovementResult<TConfig>> {
+  if (options.significance !== undefined) {
+    options = {
+      ...options,
+      significance: {
+        ...options.significance,
+        ...(options.significance.independentUnitByScenarioId === undefined
+          ? {}
+          : {
+              independentUnitByScenarioId: new Map(
+                options.significance.independentUnitByScenarioId,
+              ),
+            }),
+      },
+    }
+  }
   assertMemoryImprovementOptions(options)
   const storage = options.storage ?? fsCampaignStorage()
   const runDir = resolveRunDir(options.runDir, options.repo)

src/rag-improvement-loop.ts

@@ -152,7 +152,7 @@ export interface RunRagKnowledgeImprovementLoopOptions {
   answerQualityCostCeiling?: number
   /**
    * Makes a side-effect-free promotion decision after the library has rejected
-   * missing, regressing, unaccounted, or over-budget final evidence.
+   * missing, inconclusive, regressing, unaccounted, or over-budget final evidence.
    */
   decidePromotion?: (input: RagPromotionInput) => MaybePromise<RagPromotionResult>
   enabledPhases?: readonly RagKnowledgeImprovementPhase[]

src/rag-improvement-phases.ts

@@ -267,8 +267,12 @@ function rejectUnsafePromotionEvidence(evidence: {
     if (optimizerSource && optimizerSource.evidence !== 'observed') {
       reasons.push(`${label} optimizer package identity was not observed`)
     }
-    if (comparison.best.liftCi.low < 0) {
-      reasons.push(`${label} final comparison does not rule out a regression`)
+    if (!comparison.best.decision.promote) {
+      reasons.push(
+        comparison.best.decision.low < 0
+          ? `${label} final comparison does not rule out a regression`
+          : `${label} final comparison does not establish the required improvement`,
+      )
     }
     if (
       costCeiling !== undefined &&

src/release.ts

@@ -1,7 +1,7 @@
 import {
   type DatasetScenario,
   evaluateReleaseConfidence,
-  type GateDecision,
+  type HeldOutGateDecision,
   type ReleaseConfidenceScorecard,
   type ReleaseTraceEvidence,
   type RunRecord,
@@ -27,7 +27,7 @@ export interface KnowledgeReleaseInput {
   candidateRuns: RunRecord[]
   baselineRuns?: RunRecord[]
   traces?: ReleaseTraceEvidence[]
-  gateDecision?: GateDecision | null
+  gateDecision?: HeldOutGateDecision | null
   /** Scenario corpus used to prove train and holdout split coverage. */
   scenarios?: readonly DatasetScenario[]
   /**