[Klaud Cold] chore(evals): disable agentic evals repo-wide behind AGENTIC_EVALS_DISABLED / 全仓库停用智能体评测（AGENTIC_EVALS_DISABLED 开关）

utils/evals/EVALS.md

@@ -5,6 +5,25 @@ concurrency, kernels, and other throughput optimizations. They run separately
 from throughput; selection lives in `mark_eval_entries()` in
 `utils/matrix_logic/generate_sweep_configs.py`.
 
+## Status: agentic evals are disabled
+
+**Agentic (SWE-bench) evals are turned off repo-wide** by
+`AGENTIC_EVALS_DISABLED` in `utils/matrix_logic/generate_sweep_configs.py`.
+While it is set, no `agentic-coding` row is ever marked `run-eval`, so
+`run-sweep.yml`'s `sweep-agentic-evals` and `e2e-tests.yml`'s
+`test-sweep-agentic-evals` both find an empty matrix and skip — including
+under `--evals-only`, `--all-evals`, and the `evals-only` / `all-evals` PR
+labels. Agentic *throughput* coverage is unaffected, and fixed-sequence
+(`gsm8k`, `gpqa`, `swebench` on 8k1k) evals are unaffected.
+
+TODO(@adibarra): fix the agentic eval path and re-enable by flipping the flag
+to `False`. The selection policy below still has test coverage behind the
+`_agentic_evals_enabled()` helper in `test_generate_sweep_configs.py`, so the
+re-enable is a one-line change.
+
+The rest of this section describes the behaviour that returns when the flag is
+cleared.
+
 ## Selection
 
 - **Single-node:** 8k1k only; highest and median concurrency for every model,

utils/matrix_logic/generate_sweep_configs.py

@@ -235,6 +235,21 @@ def _multinode_parallelism_key(entry: dict) -> tuple:
     ))
 
 
+# TEMPORARY: agentic (SWE-bench) evals are disabled repo-wide.
+#
+# TODO(@adibarra): fix the agentic eval path and re-enable by flipping this to
+# False (or deleting it and the two `if AGENTIC_EVALS_DISABLED` guards below).
+# Owner is back the week of 2026-08-03; nothing else needs to change to turn
+# them back on, and no throughput coverage is affected either way.
+#
+# This is the single choke point for both dispatch paths: run-sweep.yml's
+# `sweep-agentic-evals` job reads search-space-config.agentic_evals, which
+# process_changelog.py fills from agentic-coding rows carrying run-eval: True,
+# and e2e-tests.yml's `test-sweep-agentic-evals` filters the same flag. If no
+# agentic row is ever marked run-eval, neither job has a matrix and both skip.
+AGENTIC_EVALS_DISABLED = True
+
+
 def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = False) -> list[dict]:
     """Eval selection policy:
     - Single-node: only consider 8k1k (isl=8192, osl=1024).
@@ -247,7 +262,8 @@ def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = False)
         - Ignore entries with all conc values < MIN_EVAL_CONC
         - Mark the entry containing its highest eligible concurrency
         - Set eval-conc to that highest eligible concurrency
-    - Agentic evals are opt-in to preserve default throughput coverage.
+    - Agentic evals are opt-in to preserve default throughput coverage, and are
+      currently disabled outright by AGENTIC_EVALS_DISABLED.
     """
     from collections import defaultdict
 
@@ -311,7 +327,7 @@ def _eligible_eval_concs(entry):
         mn_eval_conc[best_idx] = best_eval_conc
 
     # Default sweeps preserve every agentic throughput result.
-    if include_agentic:
+    if include_agentic and not AGENTIC_EVALS_DISABLED:
         ag_sn_groups = defaultdict(list)
         for i, entry in enumerate(matrix_values):
             if entry.get(Fields.SCENARIO_TYPE.value) != 'agentic-coding':
@@ -356,7 +372,7 @@ def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]:
 
     for entry in matrix_values:
         if entry.get(Fields.SCENARIO_TYPE.value) == 'agentic-coding':
-            if Fields.PREFILL.value not in entry:
+            if Fields.PREFILL.value not in entry and not AGENTIC_EVALS_DISABLED:
                 entry[Fields.RUN_EVAL.value] = True
             expanded_entries.append(entry)
             continue

utils/matrix_logic/test_generate_sweep_configs.py

@@ -1,7 +1,9 @@
 """Comprehensive tests for generate_sweep_configs.py"""
+import contextlib
 import pytest
 import argparse
 import copy
+import generate_sweep_configs
 from generate_sweep_configs import (
     MIN_EVAL_CONC,
     seq_len_stoi,
@@ -16,6 +18,26 @@
 )
 
 
+# =============================================================================
+# Helpers
+# =============================================================================
+
+@contextlib.contextmanager
+def _agentic_evals_enabled():
+    """Temporarily clear the AGENTIC_EVALS_DISABLED kill switch.
+
+    Agentic evals are disabled repo-wide pending TODO(@adibarra). The
+    underlying selection policy is still tested through this helper so the
+    re-enable is a one-line flip with coverage already in place.
+    """
+    previous = generate_sweep_configs.AGENTIC_EVALS_DISABLED
+    generate_sweep_configs.AGENTIC_EVALS_DISABLED = False
+    try:
+        yield
+    finally:
+        generate_sweep_configs.AGENTIC_EVALS_DISABLED = previous
+
+
 # =============================================================================
 # Test Fixtures
 # =============================================================================
@@ -227,12 +249,34 @@ def test_marks_agentic_entry_for_swebench(self):
             },
         ]
 
-        result = mark_eval_entries(matrix_values, include_agentic=True)
+        # AGENTIC_EVALS_DISABLED is a temporary repo-wide kill switch
+        # (TODO(@adibarra)); this asserts the selection policy that returns
+        # when it is flipped back off.
+        with _agentic_evals_enabled():
+            result = mark_eval_entries(matrix_values, include_agentic=True)
 
         marked = [e for e in result if e.get("run-eval")]
         assert len(marked) == 1
         assert marked[0]["conc"] == 64
 
+    def test_disable_switch_suppresses_agentic_even_when_included(self):
+        matrix_values = [
+            {
+                "scenario-type": "agentic-coding",
+                "model": "m", "runner": "b300", "framework": "vllm",
+                "precision": "fp4", "tp": 8, "conc": 64,
+            },
+        ]
+
+        assert generate_sweep_configs.AGENTIC_EVALS_DISABLED is True
+
+        result = mark_eval_entries(matrix_values, include_agentic=True)
+
+        assert [e for e in result if e.get("run-eval")] == [], (
+            "AGENTIC_EVALS_DISABLED must win over include_agentic -- "
+            "--evals-only / --all-evals must not dispatch agentic evals"
+        )
+
     def test_default_mode_does_not_mark_agentic(self):
         matrix_values = [
             {
@@ -673,12 +717,31 @@ def test_marks_agentic_entries_for_swebench(self):
             }
         ]
 
-        result = mark_all_eval_entries(entries)
+        with _agentic_evals_enabled():
+            result = mark_all_eval_entries(copy.deepcopy(entries))
 
         assert result[0]['run-eval'] is True
         assert 'eval-conc' not in result[0]
         assert 'eval-all-concs' not in result[0]
 
+    def test_disable_switch_suppresses_agentic_in_all_evals(self):
+        entries = [
+            {
+                'scenario-type': 'agentic-coding',
+                'model': 'm',
+                'runner': 'r',
+                'conc': 64,
+            }
+        ]
+
+        assert generate_sweep_configs.AGENTIC_EVALS_DISABLED is True
+
+        result = mark_all_eval_entries(entries)
+
+        assert result[0].get('run-eval') is not True, (
+            "AGENTIC_EVALS_DISABLED must win over --all-evals"
+        )
+
 
 # =============================================================================
 # Test generate_full_sweep for single-node