From 7b94ebc72aa41af63adc7aad998b86a1939ab073 Mon Sep 17 00:00:00 2001 From: Shehab Yasser Date: Fri, 3 Jul 2026 21:03:08 +0300 Subject: [PATCH 1/2] fix(harbor): instruction advertises the free baseline eval, gated on the sidecar actually granting it Two live-run findings, same root cause (the compiled instruction under-informs the optimizer about budget economics): 1. The auto_best paragraph claimed baseline evals 'spend budget without creating a candidate'. Once the sidecar grants the first baseline eval budget-free, that wording actively steers the optimizer away from the free reference measurement. Observed live: an optimizer produced three monotonically regressing candidates and never learned where zero was. 2. Two optimizers finished with nearly half their eval budget unspent. Nothing told them unspent evals are wasted or that re-measuring a noisy best candidate is a legitimate spend. The free-baseline bullet renders only when the vero tree being compiled actually has the feature (introspected from StatusSummary): the compiler and the free-baseline eval live on different PR chains, and an instruction that promises an unmetered eval the sidecar meters would send the agent to burn budget on a commit auto_best cannot select. Tests carry both arms. Co-Authored-By: Claude Opus 4.8 (1M context) --- vero/src/vero/harbor/build/compiler.py | 10 ++++ .../harbor/build/templates/instruction.md.j2 | 17 +++++- vero/tests/test_harbor_build.py | 59 +++++++++++++++++-- 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/vero/src/vero/harbor/build/compiler.py b/vero/src/vero/harbor/build/compiler.py index b667113..f5d04e7 100644 --- a/vero/src/vero/harbor/build/compiler.py +++ b/vero/src/vero/harbor/build/compiler.py @@ -8,6 +8,7 @@ from __future__ import annotations +import dataclasses import logging import re import shutil @@ -17,6 +18,7 @@ from jinja2 import Environment, FileSystemLoader from vero.harbor.build.config import BuildConfig +from vero.harbor.protocol import StatusSummary logger = logging.getLogger(__name__) @@ -278,6 +280,14 @@ def compile_task( submit_enabled=config.submit_enabled, eval_num_samples=None, bake_inner_task=bool(config.inner_task), + # The free-baseline bullet may only render when the sidecar shipping in + # this same tree actually grants the free eval; the feature lives on a + # different PR chain than the compiler, and an instruction that promises + # it without it would send the agent to burn a metered eval on a commit + # auto_best cannot select. Introspecting the protocol keeps the + # instruction truthful under any merge order. + free_baseline="free_baseline_available" + in {f.name for f in dataclasses.fields(StatusSummary)}, ) _render(jenv, "task.toml.j2", out / "task.toml", **ctx) _render(jenv, "instruction.md.j2", out / "instruction.md", **ctx) diff --git a/vero/src/vero/harbor/build/templates/instruction.md.j2 b/vero/src/vero/harbor/build/templates/instruction.md.j2 index 7761093..441de7e 100644 --- a/vero/src/vero/harbor/build/templates/instruction.md.j2 +++ b/vero/src/vero/harbor/build/templates/instruction.md.j2 @@ -19,13 +19,24 @@ progress on the splits you *are* allowed to evaluate, within a fixed budget. {% if submit_enabled %}5. When done, nominate your best commit: `vero harbor submit`.{% else %} The best commit you evaluate on `{{ selection_split }}` is selected automatically and scored on the hidden test split at the end. Only commits *other than the seeded -baseline* are selectable: evaluating the unmodified baseline spends budget without -creating a candidate, so make sure at least one eval is of a commit that contains -your changes.{% endif %} +baseline* are selectable: baseline evals create no candidate, so make sure at least +one eval is of a commit that contains your changes.{% endif %} ## Rules - Budget is finite and metered per split — spend it wisely. +{% if free_baseline %} +- Your first eval of the seeded baseline (the commit you started from) is + budget-free: once per task, not once per split, and it stays free after you have + made commits (`vero harbor eval --commit ...`). Take it on + `{{ selection_split }}` before your first candidate eval: it is the + reference score you must beat, and without it you cannot tell an improvement + from a regression. Repeat baseline evals are metered. `vero harbor status` + shows the baseline sha and whether the free eval is still available. +{% endif %} +- Scores are noisy. Unspent budget is wasted: if you finish with evals left, spend + them re-measuring your best candidate to confirm its score, or trying one more + variant, rather than stopping early. - The test split is hidden: you cannot evaluate it, and its labels never reach this container. Trying to read it will fail. - The scorer is locked. Only the eval sidecar scores. diff --git a/vero/tests/test_harbor_build.py b/vero/tests/test_harbor_build.py index f441a66..bd71265 100644 --- a/vero/tests/test_harbor_build.py +++ b/vero/tests/test_harbor_build.py @@ -4,6 +4,7 @@ from __future__ import annotations +import dataclasses import json import subprocess import tomllib @@ -13,8 +14,16 @@ import yaml from vero.harbor.build import BuildConfig, compile_task +from vero.harbor.protocol import StatusSummary from vero.harbor.serve import ServeConfig +# Whether the sidecar in THIS tree grants the budget-free first baseline eval. +# The feature and the compiler live on different PR chains; the instruction +# tests below run the arm that matches whichever chains are merged here. +_HAS_FREE_BASELINE = "free_baseline_available" in { + f.name for f in dataclasses.fields(StatusSummary) +} + def _stub_vero(root: Path) -> Path: """A minimal stand-in for the vero source tree (compiler just copies it).""" @@ -175,12 +184,52 @@ def test_instruction_warns_baseline_not_selectable(built): # baseline died with "no candidate experiments" at finalize). text = (built / "instruction.md").read_text() assert "other than the seeded" in text - assert "spends budget without" in text + assert "create no candidate" in text + + +@pytest.mark.skipif( + not _HAS_FREE_BASELINE, reason="sidecar in this tree has no free baseline eval" +) +def test_instruction_advertises_free_baseline_eval(built): + # The sidecar gives the first baseline eval away free; the instruction must + # say so or the offer goes unclaimed (found live: an optimizer produced only + # regressing candidates and never learned where zero was, because the old + # wording told it baseline evals waste budget). + text = (built / "instruction.md").read_text() + assert "budget-free" in text + assert "reference score" in text + # ...and it must aim the one-per-task freebie at the split where candidates + # are compared, or a multi-split task can waste it. + assert "once per task" in text + + +@pytest.mark.skipif( + _HAS_FREE_BASELINE, reason="sidecar in this tree grants the free baseline eval" +) +def test_instruction_omits_free_baseline_claim_when_unsupported(built): + # Merge-order guard: if the compiler chain lands without the free-baseline + # chain, the instruction must not promise a freebie the sidecar will meter; + # acting on that promise burns a metered eval on a commit auto_best cannot + # select (fatal on a run_budget=1 task). + text = (built / "instruction.md").read_text() + assert "budget-free" not in text + + +def test_instruction_tells_agent_to_spend_whole_budget(built): + # Two live runs ended with nearly half the eval budget unspent; the + # instruction must state that unspent evals are wasted and re-measurement + # is a legitimate spend. + text = (built / "instruction.md").read_text() + assert "Unspent budget is wasted" in text + assert "re-measuring your best candidate" in text def test_submit_mode_instruction_has_no_baseline_warning(tmp_path, monkeypatch): - # The warning belongs to the auto_best branch only; pin the conditional - # boundary so a template refactor cannot leak it into submit-mode tasks. + # The not-selectable warning belongs to the auto_best branch only; pin the + # conditional boundary so a template refactor cannot leak it into + # submit-mode tasks. The free-baseline and spend-the-budget rules are + # mode-agnostic (metering does not depend on the selection mode) and must + # survive in both. monkeypatch.setenv("VERO_SKIP_SECRET_CHECK", "1") config = BuildConfig( name="vero/gsm8k-opt", @@ -195,4 +244,6 @@ def test_submit_mode_instruction_has_no_baseline_warning(tmp_path, monkeypatch): out = compile_task(config, tmp_path / "task", vero_root=_stub_vero(tmp_path)) text = (out / "instruction.md").read_text() assert "other than the seeded" not in text - assert "spends budget without" not in text + assert "create no candidate" not in text + assert ("budget-free" in text) == _HAS_FREE_BASELINE + assert "Unspent budget is wasted" in text From 38f1195ca5395d57f9ab24686e1885820df707eb Mon Sep 17 00:00:00 2001 From: Shehab Yasser Date: Sat, 4 Jul 2026 09:01:39 +0300 Subject: [PATCH 2/2] refactor(harbor): name the free-baseline protocol-field coupling as a constant The compiler introspects StatusSummary for the free-baseline field name; that string was an unnamed literal, the sole coupling to the field PR #25 adds on a separate chain. Hoist it to _FREE_BASELINE_FIELD so the compiler<->protocol contract has one documented source. A hard import-time assert cannot live here: the field is legitimately absent on this branch's base until the sidecar PR merges, which is why the render is introspection-gated in the first place. Addresses Greptile P2 on this PR. Co-Authored-By: Claude Opus 4.8 (1M context) --- vero/src/vero/harbor/build/compiler.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/vero/src/vero/harbor/build/compiler.py b/vero/src/vero/harbor/build/compiler.py index f5d04e7..0575ba1 100644 --- a/vero/src/vero/harbor/build/compiler.py +++ b/vero/src/vero/harbor/build/compiler.py @@ -24,6 +24,13 @@ _TEMPLATES = Path(__file__).parent / "templates" +# The StatusSummary field the sidecar sets to signal it grants the agent a free +# baseline eval. The compiler renders the free-baseline instruction bullet only +# when this field exists on the protocol shipping in the same tree (see the +# free_baseline ctx entry). Kept as a named constant so the compiler<->protocol +# coupling has one source: rename this and StatusSummary. together. +_FREE_BASELINE_FIELD = "free_baseline_available" + # Container paths (must match the templates). VERO_DIR = "/opt/vero" AGENT_BASELINE = "/opt/agent-baseline" # sidecar engine workspace @@ -285,8 +292,11 @@ def compile_task( # different PR chain than the compiler, and an instruction that promises # it without it would send the agent to burn a metered eval on a commit # auto_best cannot select. Introspecting the protocol keeps the - # instruction truthful under any merge order. - free_baseline="free_baseline_available" + # instruction truthful under any merge order. _FREE_BASELINE_FIELD is the + # single source of the coupled field name: rename StatusSummary. and + # this constant together (they cannot import-assert each other, since the + # field is absent on this branch's base until the sidecar PR merges). + free_baseline=_FREE_BASELINE_FIELD in {f.name for f in dataclasses.fields(StatusSummary)}, ) _render(jenv, "task.toml.j2", out / "task.toml", **ctx)