feat: Verify Bugbot findings against repository evidence before publication (#950)

.github/workflows/ci_check.yml

@@ -5,8 +5,6 @@ on:
     branches: [master]
   pull_request:
     types: [opened, synchronize]
-  merge_group:
-    types: [checks_requested]
   workflow_dispatch:
 
 concurrency:
@@ -17,57 +15,122 @@ defaults:
   run:
     shell: bash
 
+permissions:
+  contents: read
+
 jobs:
-  ci-check:
-    name: CI Check
-    # A merge group can contain fork code without exposing the originating PR here.
-    runs-on: ${{ fromJSON((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository)) && '["ubuntu-latest"]' || '["self-hosted","codex","Linux"]') }}
-    timeout-minutes: 20
-    permissions:
-      contents: read
+  static-checks:
+    name: CI / Static checks
+    if: &source-ci ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
+    runs-on: &source-runner
+      group: vypdev
+      labels: [self-hosted, codex, Linux]
+    timeout-minutes: 10
     steps:
-      - name: Verify Ubuntu runner
+      - &verify-ubuntu
+        name: Verify Ubuntu runner
         run: . /etc/os-release && test "$ID" = ubuntu
-      - uses: actions/checkout@v5
+
+      - &checkout
+        uses: actions/checkout@v5
         with:
           persist-credentials: false
           # Contract tests inspect the immutable in-repository action revision
           # used by distributed workflows, so its commit must be available.
           fetch-depth: 0
 
-      - name: Set up pnpm
-        # Avoid the standalone @pnpm/exe binary, which has no working Intel
-        # macOS build for this pnpm line.
+      - name: Validate Git diff
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PUSH_BEFORE_SHA: ${{ github.event.before }}
+        run: |
+          set -euo pipefail
+          case "$EVENT_NAME" in
+            pull_request) base="$PR_BASE_SHA" ;;
+            push) base="$PUSH_BEFORE_SHA" ;;
+            workflow_dispatch) base="$(git rev-parse HEAD^)" ;;
+            *) echo "Unsupported diff-check event" >&2; exit 1 ;;
+          esac
+          if [[ "$EVENT_NAME" == push && "$base" =~ ^0{40}$ ]]; then
+            base="$(git hash-object -t tree --stdin </dev/null)"
+          fi
+          [[ "$base" =~ ^[0-9a-f]{40}$ ]] || { echo "Missing diff-check base commit" >&2; exit 1; }
+          git -c core.whitespace=blank-at-eol,blank-at-eof,space-before-tab,cr-at-eol diff --check "$base" HEAD -- . ':(exclude)build/**'
+
+      - &pnpm
+        name: Set up pnpm
         uses: pnpm/action-setup@v5
         with:
           version: 10.12.4
           standalone: false
           dest: ${{ runner.temp }}/setup-pnpm-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }}
 
-      - name: Set up Node.js 24
+      - &node
+        name: Set up Node.js 24
         uses: actions/setup-node@v7
         with:
           node-version: '24.x'
-          cache: pnpm
+          # Each self-hosted runner can reuse its local pnpm store.
+          package-manager-cache: false
 
+      - &runner-scratch
+        name: Use runner scratch for CI
+        # Self-hosted runners can mount /tmp on a smaller filesystem than the work directory.
+        run: printf 'TMPDIR=%s\n' "$RUNNER_TEMP" >> "$GITHUB_ENV"
 
-      - name: Install dependencies
+      - &install
+        name: Install dependencies
         run: pnpm install --frozen-lockfile
 
-      - name: Validate generated guidance checkout
-        run: |
-          git ls-files --eol -- .copilot/repository-profile.json .copilot/AGENT_GUIDE.md .agents/skills/copilot-repository-workflow/SKILL.md
-          pnpm run validate:agent-docs
+      - name: Lint
+        run: pnpm run lint
+
+      - name: Typecheck
+        run: pnpm run typecheck
+
+      - name: Validate documentation assets
+        run: pnpm run validate:docs-page
+
+      - name: Validate documentation contract
+        run: pnpm run validate:documentation
+
+      - name: Validate workflow contract
+        run: pnpm run validate:workflows
+
+      - name: Validate product specification catalog
+        run: pnpm run validate:specifications
+
+      - name: Validate setup acceptance ledger
+        run: pnpm run validate:setup-acceptance
 
       - name: Audit production dependencies
         run: pnpm audit --prod --audit-level=high
 
+      - name: Validate Bugbot quality benchmark
+        run: pnpm run eval:bugbot
+
+      - name: Validate Bugbot shadow verification
+        run: pnpm run test:bugbot-shadow-verification
+
+  build-package:
+    name: CI / Build and package
+    if: *source-ci
+    runs-on: *source-runner
+    timeout-minutes: 10
+    steps:
+      - *verify-ubuntu
+      - *checkout
+      - *pnpm
+      - *node
+      - *runner-scratch
+      - *install
+
       - name: Build
         run: pnpm run build
 
       # Checked-in ncc bundles are generated on Unix; Windows output differs.
       - name: Validate generated bundles
-        if: runner.os != 'Windows'
         run: pnpm run validate:build
 
       - name: Validate extension architecture and reproducibility
@@ -79,19 +142,33 @@ jobs:
       - name: Smoke test npm package
         run: pnpm run smoke:npm-package
 
-      - name: Typecheck
-        run: pnpm run typecheck
+  coverage:
+    name: CI / Tests and coverage
+    if: *source-ci
+    runs-on: *source-runner
+    timeout-minutes: 25
+    steps:
+      - *verify-ubuntu
+      - *checkout
+      - *pnpm
+      - *node
+      - *runner-scratch
+      - *install
+
+      - name: Validate generated guidance checkout
+        run: |
+          git ls-files --eol -- .copilot/repository-profile.json .copilot/AGENT_GUIDE.md .agents/skills/copilot-repository-workflow/SKILL.md
+          pnpm run validate:agent-docs
 
       - name: Run tests with coverage
         run: pnpm run test:coverage
 
-      - name: Validate Bugbot quality benchmark
-        run: pnpm run eval:bugbot
+      - name: Verify tests preserved generated guidance
+        run: pnpm run validate:agent-docs
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v6
-        # Coverage reporting is advisory; Codecov v6 requires gpg even when
-        # fail_ci_if_error is false, and the internal macOS runner may not have it.
+        # Coverage reporting is advisory; the local coverage gates above are required.
         continue-on-error: true
         with:
           files: ./coverage/lcov.info,./extension/coverage/lcov.info
@@ -100,44 +177,23 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
 
-      - name: Lint
-        run: pnpm run lint
-
-      - name: Validate documentation
-        run: pnpm run validate:agent-docs
-
-      - name: Validate documentation assets
-        run: pnpm run validate:docs-page
-
-      - name: Validate documentation contract
-        run: pnpm run validate:documentation
-
-      - name: Validate workflow contract
-        run: pnpm run validate:workflows
-
-      - name: Validate product specification catalog
-        run: pnpm run validate:specifications
-
-      - name: Validate setup acceptance ledger
-        run: pnpm run validate:setup-acceptance
-
-      - name: Validate Git diff
+  ci-check:
+    # Keep the ruleset and approval consumer bound to this exact check name.
+    name: CI Check
+    if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
+    needs: [static-checks, build-package, coverage]
+    runs-on: *source-runner
+    timeout-minutes: 2
+    permissions: {}
+    steps:
+      - name: Require all CI jobs to succeed
         env:
-          EVENT_NAME: ${{ github.event_name }}
-          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
-          MERGE_BASE_SHA: ${{ github.event.merge_group.base_sha }}
-          PUSH_BEFORE_SHA: ${{ github.event.before }}
+          STATIC_RESULT: ${{ needs.static-checks.result }}
+          BUILD_RESULT: ${{ needs.build-package.result }}
+          COVERAGE_RESULT: ${{ needs.coverage.result }}
         run: |
-          set -euo pipefail
-          case "$EVENT_NAME" in
-            pull_request) base="$PR_BASE_SHA" ;;
-            merge_group) base="$MERGE_BASE_SHA" ;;
-            push) base="$PUSH_BEFORE_SHA" ;;
-            workflow_dispatch) base="$(git rev-parse HEAD^)" ;;
-            *) echo "Unsupported diff-check event" >&2; exit 1 ;;
-          esac
-          if [[ "$EVENT_NAME" == push && "$base" =~ ^0{40}$ ]]; then
-            base="$(git hash-object -t tree --stdin </dev/null)"
-          fi
-          [[ "$base" =~ ^[0-9a-f]{40}$ ]] || { echo "Missing diff-check base commit" >&2; exit 1; }
-          git -c core.whitespace=blank-at-eol,blank-at-eof,space-before-tab,cr-at-eol diff --check "$base" HEAD -- . ':(exclude)build/**'
+          printf '%s\n' '| Job | Result |' '| --- | --- |' \
+            "| Static checks | $STATIC_RESULT |" \
+            "| Build and package | $BUILD_RESULT |" \
+            "| Tests and coverage | $COVERAGE_RESULT |" | tee -a "$GITHUB_STEP_SUMMARY"
+          [[ "$STATIC_RESULT" == success && "$BUILD_RESULT" == success && "$COVERAGE_RESULT" == success ]]

.github/workflows/copilot_comment_runtime.yml

@@ -44,9 +44,10 @@ jobs:
   copilot:
     name: Copilot
     if: ${{ (vars.COPILOT_BOT_LOGIN == '' || github.actor != vars.COPILOT_BOT_LOGIN) && (github.event_name == 'issue_comment' || (github.event_name == 'pull_request_review_comment' && github.event.pull_request.head.repo.full_name == github.repository)) }}
-    runs-on: ${{ fromJSON(inputs.runs-on) }}
+    runs-on: ${{ fromJSON(github.repository == 'vypdev/copilot' && '["self-hosted","codex","Linux"]' || inputs['runs-on']) }}
     timeout-minutes: 120
     permissions:
+      actions: read
       checks: write
       contents: read
     env:
@@ -104,6 +105,11 @@ jobs:
           bugbot-severity: ${{ vars.BUGBOT_SEVERITY || 'low' }}
           bugbot-comment-limit: ${{ vars.BUGBOT_COMMENT_LIMIT || '20' }}
           bugbot-fix-verify-commands: ${{ vars.BUGBOT_AUTOFIX_VERIFY_COMMANDS }}
+          bugbot-verification-mode: ${{ vars.BUGBOT_VERIFICATION_MODE || 'disabled' }}
+          bugbot-verifier-provider: ${{ vars.BUGBOT_VERIFIER_PROVIDER }}
+          bugbot-verifier-model-provider: ${{ vars.BUGBOT_VERIFIER_MODEL_PROVIDER }}
+          bugbot-verifier-model: ${{ vars.BUGBOT_VERIFIER_MODEL }}
+          bugbot-verifier-effort: ${{ vars.BUGBOT_VERIFIER_EFFORT }}
           bugbot-dry-run: ${{ vars.BUGBOT_DRY_RUN || 'false' }}
           bugbot-effort: ${{ vars.BUGBOT_EFFORT || 'smart' }}
           bugbot-review-drafts: ${{ vars.BUGBOT_REVIEW_DRAFTS || 'false' }}
@@ -162,7 +168,7 @@ jobs:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
           OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
-          CURSOR_API_KEY: ${{ (vars.AGENT_PROVIDER == 'cursor' || vars.FINDINGS_PROVIDER == 'cursor' || vars.FIXER_PROVIDER == 'cursor' || vars.PLANNER_PROVIDER == 'cursor' || vars.REVIEWER_PROVIDER == 'cursor' || vars.TESTER_PROVIDER == 'cursor') && secrets.CURSOR_API_KEY || '' }}
+          CURSOR_API_KEY: ${{ (vars.AGENT_PROVIDER == 'cursor' || vars.FINDINGS_PROVIDER == 'cursor' || vars.FIXER_PROVIDER == 'cursor' || vars.PLANNER_PROVIDER == 'cursor' || vars.REVIEWER_PROVIDER == 'cursor' || vars.TESTER_PROVIDER == 'cursor' || (vars.BUGBOT_VERIFICATION_MODE == 'shadow' && vars.BUGBOT_VERIFIER_PROVIDER == 'cursor')) && secrets.CURSOR_API_KEY || '' }}
           CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }}
           COPILOT_EVIDENCE_TOKEN: ${{ github.token }}
       - name: Prepare repository verification 1

.github/workflows/copilot_commit.yml

@@ -23,6 +23,7 @@ jobs:
       group: copilot-push-${{ github.repository }}-${{ github.ref_name }}
       cancel-in-progress: true
     permissions:
+      actions: read
       checks: write
       contents: read
     steps:
@@ -58,6 +59,11 @@ jobs:
           bugbot-comment-limit: ${{ vars.BUGBOT_COMMENT_LIMIT || '20' }}
           bugbot-fix-verify-commands: ${{ vars.BUGBOT_AUTOFIX_VERIFY_COMMANDS }}
           bugbot-dry-run: ${{ vars.BUGBOT_DRY_RUN || 'false' }}
+          bugbot-verification-mode: ${{ vars.BUGBOT_VERIFICATION_MODE || 'disabled' }}
+          bugbot-verifier-provider: ${{ vars.BUGBOT_VERIFIER_PROVIDER }}
+          bugbot-verifier-model-provider: ${{ vars.BUGBOT_VERIFIER_MODEL_PROVIDER }}
+          bugbot-verifier-model: ${{ vars.BUGBOT_VERIFIER_MODEL }}
+          bugbot-verifier-effort: ${{ vars.BUGBOT_VERIFIER_EFFORT }}
           bugbot-effort: ${{ vars.BUGBOT_EFFORT || 'smart' }}
           bugbot-review-drafts: ${{ vars.BUGBOT_REVIEW_DRAFTS || 'false' }}
           bugbot-trace-rules: ${{ vars.BUGBOT_TRACE_RULES || 'false' }}
@@ -96,5 +102,5 @@ jobs:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
           OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
-          CURSOR_API_KEY: ${{ (vars.AGENT_PROVIDER == 'cursor' || vars.FINDINGS_PROVIDER == 'cursor' || vars.FIXER_PROVIDER == 'cursor' || vars.PLANNER_PROVIDER == 'cursor' || vars.REVIEWER_PROVIDER == 'cursor' || vars.TESTER_PROVIDER == 'cursor') && secrets.CURSOR_API_KEY || '' }}
+          CURSOR_API_KEY: ${{ (vars.AGENT_PROVIDER == 'cursor' || vars.FINDINGS_PROVIDER == 'cursor' || vars.FIXER_PROVIDER == 'cursor' || vars.PLANNER_PROVIDER == 'cursor' || vars.REVIEWER_PROVIDER == 'cursor' || vars.TESTER_PROVIDER == 'cursor' || (vars.BUGBOT_VERIFICATION_MODE == 'shadow' && vars.BUGBOT_VERIFIER_PROVIDER == 'cursor')) && secrets.CURSOR_API_KEY || '' }}
           CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }}

.github/workflows/copilot_credential_health.yml

@@ -51,7 +51,7 @@ jobs:
   verify-pat:
     if: ${{ inputs.check_pat }}
     name: Verify PAT
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, codex, Linux]
     timeout-minutes: 10
     steps:
       - name: Validate PAT without exposing its value
@@ -69,7 +69,7 @@ jobs:
   verify-openai:
     if: ${{ inputs.check_openai }}
     name: Verify OPENAI_API_KEY
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, codex, Linux]
     timeout-minutes: 10
     steps:
       - name: Validate OpenAI key without exposing its value
@@ -86,7 +86,7 @@ jobs:
   verify-anthropic:
     if: ${{ inputs.check_anthropic }}
     name: Verify ANTHROPIC_API_KEY
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, codex, Linux]
     timeout-minutes: 10
     steps:
       - name: Validate Anthropic key without exposing its value
@@ -103,7 +103,7 @@ jobs:
   verify-google:
     if: ${{ inputs.check_google }}
     name: Verify GOOGLE_API_KEY
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, codex, Linux]
     timeout-minutes: 10
     steps:
       - name: Validate Google key without exposing its value
@@ -120,7 +120,7 @@ jobs:
   verify-openrouter:
     if: ${{ inputs.check_openrouter }}
     name: Verify OPENROUTER_API_KEY
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, codex, Linux]
     timeout-minutes: 10
     steps:
       - name: Validate OpenRouter key without exposing its value
@@ -137,7 +137,7 @@ jobs:
   verify-cursor:
     if: ${{ inputs.check_cursor }}
     name: Verify CURSOR_API_KEY
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, codex, Linux]
     timeout-minutes: 10
     steps:
       - name: Validate Cursor key without exposing its value
@@ -156,7 +156,7 @@ jobs:
   verify-opencode:
     if: ${{ inputs.check_opencode }}
     name: Verify OPENCODE_API_KEY
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, codex, Linux]
     timeout-minutes: 10
     steps:
       - name: Validate OpenCode key without exposing its value
@@ -173,7 +173,7 @@ jobs:
   verify-codex-api-key:
     if: ${{ inputs.check_codex_api_key }}
     name: Verify CODEX_API_KEY
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, codex, Linux]
     timeout-minutes: 10
     steps:
       - name: Validate Codex API key without exposing its value

.github/workflows/copilot_issue.yml

@@ -65,7 +65,7 @@ jobs:
           issue-managed-branches: true
           pre-branch-sdd: true
           verification-setup: ${{ vars.VERIFICATION_SETUP || 'false' }}
-          ai-ignore-files: build/*,specs/CATALOG.md
+          ai-ignore-files: build/*,specs/CATALOG.md,quality/bugbot/evidence/issue-*/*.jsonl,quality/bugbot/evidence/issue-*/*.log.gz
           bugbot-dry-run: ${{ vars.BUGBOT_DRY_RUN || 'false' }}
           bugbot-effort: ${{ vars.BUGBOT_EFFORT || 'smart' }}
           bugbot-review-drafts: ${{ vars.BUGBOT_REVIEW_DRAFTS || 'false' }}