diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index 5d07f887f6..8247138a1b 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -5,6 +5,25 @@ concurrency, kernels, and other throughput optimizations. They run separately from throughput; selection lives in `mark_eval_entries()` in `utils/matrix_logic/generate_sweep_configs.py`. +## Status: agentic evals are disabled + +**Agentic (SWE-bench) evals are turned off repo-wide** by +`AGENTIC_EVALS_DISABLED` in `utils/matrix_logic/generate_sweep_configs.py`. +While it is set, no `agentic-coding` row is ever marked `run-eval`, so +`run-sweep.yml`'s `sweep-agentic-evals` and `e2e-tests.yml`'s +`test-sweep-agentic-evals` both find an empty matrix and skip — including +under `--evals-only`, `--all-evals`, and the `evals-only` / `all-evals` PR +labels. Agentic *throughput* coverage is unaffected, and fixed-sequence +(`gsm8k`, `gpqa`, `swebench` on 8k1k) evals are unaffected. + +TODO(@adibarra): fix the agentic eval path and re-enable by flipping the flag +to `False`. The selection policy below still has test coverage behind the +`_agentic_evals_enabled()` helper in `test_generate_sweep_configs.py`, so the +re-enable is a one-line change. + +The rest of this section describes the behaviour that returns when the flag is +cleared. + ## Selection - **Single-node:** 8k1k only; highest and median concurrency for every model, diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 7df1a75b64..91d207e993 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -235,6 +235,21 @@ def _multinode_parallelism_key(entry: dict) -> tuple: )) +# TEMPORARY: agentic (SWE-bench) evals are disabled repo-wide. +# +# TODO(@adibarra): fix the agentic eval path and re-enable by flipping this to +# False (or deleting it and the two `if AGENTIC_EVALS_DISABLED` guards below). +# Owner is back the week of 2026-08-03; nothing else needs to change to turn +# them back on, and no throughput coverage is affected either way. +# +# This is the single choke point for both dispatch paths: run-sweep.yml's +# `sweep-agentic-evals` job reads search-space-config.agentic_evals, which +# process_changelog.py fills from agentic-coding rows carrying run-eval: True, +# and e2e-tests.yml's `test-sweep-agentic-evals` filters the same flag. If no +# agentic row is ever marked run-eval, neither job has a matrix and both skip. +AGENTIC_EVALS_DISABLED = True + + def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = False) -> list[dict]: """Eval selection policy: - Single-node: only consider 8k1k (isl=8192, osl=1024). @@ -247,7 +262,8 @@ def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = False) - Ignore entries with all conc values < MIN_EVAL_CONC - Mark the entry containing its highest eligible concurrency - Set eval-conc to that highest eligible concurrency - - Agentic evals are opt-in to preserve default throughput coverage. + - Agentic evals are opt-in to preserve default throughput coverage, and are + currently disabled outright by AGENTIC_EVALS_DISABLED. """ from collections import defaultdict @@ -311,7 +327,7 @@ def _eligible_eval_concs(entry): mn_eval_conc[best_idx] = best_eval_conc # Default sweeps preserve every agentic throughput result. - if include_agentic: + if include_agentic and not AGENTIC_EVALS_DISABLED: ag_sn_groups = defaultdict(list) for i, entry in enumerate(matrix_values): if entry.get(Fields.SCENARIO_TYPE.value) != 'agentic-coding': @@ -356,7 +372,7 @@ def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]: for entry in matrix_values: if entry.get(Fields.SCENARIO_TYPE.value) == 'agentic-coding': - if Fields.PREFILL.value not in entry: + if Fields.PREFILL.value not in entry and not AGENTIC_EVALS_DISABLED: entry[Fields.RUN_EVAL.value] = True expanded_entries.append(entry) continue diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 1ad4e3c922..e10b46ec7f 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -1,7 +1,9 @@ """Comprehensive tests for generate_sweep_configs.py""" +import contextlib import pytest import argparse import copy +import generate_sweep_configs from generate_sweep_configs import ( MIN_EVAL_CONC, seq_len_stoi, @@ -16,6 +18,26 @@ ) +# ============================================================================= +# Helpers +# ============================================================================= + +@contextlib.contextmanager +def _agentic_evals_enabled(): + """Temporarily clear the AGENTIC_EVALS_DISABLED kill switch. + + Agentic evals are disabled repo-wide pending TODO(@adibarra). The + underlying selection policy is still tested through this helper so the + re-enable is a one-line flip with coverage already in place. + """ + previous = generate_sweep_configs.AGENTIC_EVALS_DISABLED + generate_sweep_configs.AGENTIC_EVALS_DISABLED = False + try: + yield + finally: + generate_sweep_configs.AGENTIC_EVALS_DISABLED = previous + + # ============================================================================= # Test Fixtures # ============================================================================= @@ -227,12 +249,34 @@ def test_marks_agentic_entry_for_swebench(self): }, ] - result = mark_eval_entries(matrix_values, include_agentic=True) + # AGENTIC_EVALS_DISABLED is a temporary repo-wide kill switch + # (TODO(@adibarra)); this asserts the selection policy that returns + # when it is flipped back off. + with _agentic_evals_enabled(): + result = mark_eval_entries(matrix_values, include_agentic=True) marked = [e for e in result if e.get("run-eval")] assert len(marked) == 1 assert marked[0]["conc"] == 64 + def test_disable_switch_suppresses_agentic_even_when_included(self): + matrix_values = [ + { + "scenario-type": "agentic-coding", + "model": "m", "runner": "b300", "framework": "vllm", + "precision": "fp4", "tp": 8, "conc": 64, + }, + ] + + assert generate_sweep_configs.AGENTIC_EVALS_DISABLED is True + + result = mark_eval_entries(matrix_values, include_agentic=True) + + assert [e for e in result if e.get("run-eval")] == [], ( + "AGENTIC_EVALS_DISABLED must win over include_agentic -- " + "--evals-only / --all-evals must not dispatch agentic evals" + ) + def test_default_mode_does_not_mark_agentic(self): matrix_values = [ { @@ -673,12 +717,31 @@ def test_marks_agentic_entries_for_swebench(self): } ] - result = mark_all_eval_entries(entries) + with _agentic_evals_enabled(): + result = mark_all_eval_entries(copy.deepcopy(entries)) assert result[0]['run-eval'] is True assert 'eval-conc' not in result[0] assert 'eval-all-concs' not in result[0] + def test_disable_switch_suppresses_agentic_in_all_evals(self): + entries = [ + { + 'scenario-type': 'agentic-coding', + 'model': 'm', + 'runner': 'r', + 'conc': 64, + } + ] + + assert generate_sweep_configs.AGENTIC_EVALS_DISABLED is True + + result = mark_all_eval_entries(entries) + + assert result[0].get('run-eval') is not True, ( + "AGENTIC_EVALS_DISABLED must win over --all-evals" + ) + # ============================================================================= # Test generate_full_sweep for single-node