diff --git a/README.md b/README.md index 4482b51..539eae0 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ VLM = image + text, and ALM = audio + text; ✓ marks a configured modality. | Gemma 4 | `gemma-4-e2b`, `gemma-4-e4b`, `gemma-4-12b` | ✓ | ✓ | ✓ | | Nemotron 3 Nano | `nemotron-3-nano-4b` | ✓ | — | — | | Nemotron 3 Nano Omni | `nemotron-3-nano-omni-30b-a3b` | — | ✓ | ✓ | -| Gemini 3.x (API) | `gemini-3.7-flash`, `gemini-3.6-flash`, `gemini-3.5-flash`, `gemini-3.5-flash-lite`, `gemini-3.1-flash-lite` | ✓ | ✓ | ✓ | +| Gemini 3.x (API) | `gemini-3.7-flash`, `gemini-3.6-flash`, `gemini-3.5-flash`, `gemini-3.5-flash-lite`, `gemini-3.1-flash-lite`, `gemini-3.1-pro-preview` | ✓ | ✓ | ✓ | | Gemini 2.5 (API) | `gemini-2.5-flash`, `gemini-2.5-flash-lite`, `gemini-2.5-pro` | ✓ | ✓ | ✓ | Open-weight models use an OpenAI-compatible server (`--backend endpoint`, diff --git a/docs/index.html b/docs/index.html index 414e973..c4219d2 100644 --- a/docs/index.html +++ b/docs/index.html @@ -591,7 +591,7 @@

A self-improving loop that repairs open models automatically — and verifie Documentation PyPI ↗ -

pip install evalrx · CC0-1.0 · github.com/evalvitals/evalrx

+

pip install evalrx · PolyForm Noncommercial 1.0.0 · github.com/evalvitals/evalrx

@@ -1306,7 +1306,7 @@

Point it at your eval logs.

License -
EvalRx — model health for open-weight models: diagnosed, repaired, verified. CC0-1.0.
+
EvalRx — model health for open-weight models: diagnosed, repaired, verified. PolyForm Noncommercial 1.0.0.
diff --git a/evalrx/models/backends/gemini_compat.py b/evalrx/models/backends/gemini_compat.py index 4438878..f3d8278 100644 --- a/evalrx/models/backends/gemini_compat.py +++ b/evalrx/models/backends/gemini_compat.py @@ -14,8 +14,8 @@ diagnosis run: * **Thinking floor.** The benchmark runs every model with thinking as far OFF - as the model allows. 3.x models take ``thinking_level`` (3.7-flash bottoms - out at ``low``, the others at ``minimal``); 2.5 models take + as the model allows. 3.x models take ``thinking_level`` (3.7-flash and + 3.1-pro-preview bottom out at ``low``, the others at ``minimal``); 2.5 models take ``thinking_budget`` (``0`` on flash / flash-lite; 2.5-pro cannot switch it off, floor 128). :func:`thinking_config` resolves the floor per model id, ``ThinkingPolicy(level=..., budget=...)`` overrides it, and a model that @@ -74,6 +74,7 @@ "gemini-3.5-flash": "minimal", "gemini-3.5-flash-lite": "minimal", "gemini-3.1-flash-lite": "minimal", + "gemini-3.1-pro-preview": "low", # thinking cannot be disabled } #: Lowest ``thinking_budget`` each 2.5 model accepts (0 = thinking off). BUDGET_FLOOR: dict[str, int] = { diff --git a/evalrx/specs.py b/evalrx/specs.py index b16b249..c1aae73 100755 --- a/evalrx/specs.py +++ b/evalrx/specs.py @@ -698,8 +698,8 @@ def list_specs() -> list[str]: # empty, ``APIModel`` sends ``spec.hf_repo or spec.key``). Every model below # lists text, image, video, audio and PDF as inputs on its model card, so one # spec serves the llm / vlm / alm cells. Thinking: the 3.x models expose -# ``thinking_level`` (3.7-flash bottoms out at ``low``, the rest at -# ``minimal``); the 2.5 models expose ``thinking_budget`` (0 = off on flash / +# ``thinking_level`` (3.7-flash and 3.1-pro-preview bottom out at ``low``, +# the rest at ``minimal``); the 2.5 models expose ``thinking_budget`` (0 = off on flash / # flash-lite; 2.5-pro cannot switch it off, floor 128) — the runtime sends the # floor unless told otherwise. Logprobs: none for 3.x ("working as intended", # Google forum 2026-08-05) and withdrawn on 2.5, so the backend is GENERATE-only @@ -710,6 +710,7 @@ def list_specs() -> list[str]: ("gemini-3.5-flash", "thinking_level floor 'minimal'"), ("gemini-3.5-flash-lite", "thinking_level floor 'minimal' (its default)"), ("gemini-3.1-flash-lite", "thinking_level floor 'minimal' (levels per the card; floor unverified)"), + ("gemini-3.1-pro-preview", "thinking_level floor 'low' (thinking cannot be disabled)"), ("gemini-2.5-flash", "thinking_budget 0 turns thinking off (default on)"), ("gemini-2.5-flash-lite", "thinking_budget 0 (its default)"), ("gemini-2.5-pro", "thinking cannot be disabled (budget floor 128)"), diff --git a/examples/benchmark/README.md b/examples/benchmark/README.md index 9414049..a6f930f 100644 --- a/examples/benchmark/README.md +++ b/examples/benchmark/README.md @@ -39,12 +39,18 @@ differs (Qwen3.5 text tower vs vision tower); Gemma 4 is one spec for all three. | `gemini-3.5-flash` | Gemini (API) | ✓ | ✓ | ✓ | 0 | `gemini-3.5-flash` | | `gemini-3.5-flash-lite` | Gemini (API) | ✓ | ✓ | ✓ | 0 | `gemini-3.5-flash-lite` | | `gemini-3.1-flash-lite` | Gemini (API) | ✓ | ✓ | ✓ | 0 | `gemini-3.1-flash-lite` | +| `gemini-3.1-pro-preview` | Gemini (API) | ✓ | ✓ | ✓ | 0 | `gemini-3.1-pro-preview` (thinking floor `low`) | | `gemini-2.5-flash` | Gemini (API) | ✓ | ✓ | ✓ | 0 | `gemini-2.5-flash` (`thinking_budget` 0) | | `gemini-2.5-flash-lite` | Gemini (API) | ✓ | ✓ | ✓ | 0 | `gemini-2.5-flash-lite` (`thinking_budget` 0) | | `gemini-2.5-pro` | Gemini (API) | ✓ | ✓ | ✓ | 0 | `gemini-2.5-pro` (thinking cannot be disabled; budget 128) | `python -m _common.run --list` prints the same table from the code. +Gemini 3.1 Pro uses the official API id +[`gemini-3.1-pro-preview`](https://ai.google.dev/gemini-api/docs/models/gemini-3.1-pro-preview). +Its minimum thinking level is `low`; thinking cannot be disabled. This model +is configured for all three modalities; live benchmark validation is pending. + Gemma 4 12B **does** take audio (the model card lists audio on E2B, E4B and 12B; the 12B is the encoder-free "Unified" variant), so it sits in the ALM row; drop it from `SIZES["gemma-4-12b"].specs` if it should stay out. @@ -133,8 +139,9 @@ the final significance gate, and the validation data costs no confirm power. all default ON otherwise); `--enable-thinking` flips it for one run. Gemini cannot always switch thinking off, so the runtime sends each model's **floor**: `thinking_level=minimal` on 3.6/3.5/3.5-lite/3.1-lite, `low` on - 3.7-flash (its lowest), `thinking_budget=0` on 2.5-flash / flash-lite and 128 - on 2.5-pro. `--thinking-level {minimal,low,medium,high}` / `--thinking-budget N` + 3.7-flash and 3.1-pro-preview (their lowest), `thinking_budget=0` on + 2.5-flash / flash-lite and 128 on 2.5-pro. + `--thinking-level {minimal,low,medium,high}` / `--thinking-budget N` name a setting explicitly; `--enable-thinking` leaves the API default. A model that rejects the config falls back to the default once, logged, and the served `model_version` is written to `baseline.json` because a stable id is diff --git a/examples/benchmark/_common/models.py b/examples/benchmark/_common/models.py index c6f733e..f48c84b 100644 --- a/examples/benchmark/_common/models.py +++ b/examples/benchmark/_common/models.py @@ -67,7 +67,8 @@ def default_device(self) -> str: #: image, video and audio, so each size fills all three modality cells. _GEMINI = ( "gemini-3.7-flash", "gemini-3.6-flash", "gemini-3.5-flash", "gemini-3.5-flash-lite", - "gemini-3.1-flash-lite", "gemini-2.5-flash", "gemini-2.5-flash-lite", "gemini-2.5-pro", + "gemini-3.1-flash-lite", "gemini-3.1-pro-preview", + "gemini-2.5-flash", "gemini-2.5-flash-lite", "gemini-2.5-pro", ) SIZES: dict[str, Size] = { diff --git a/examples/benchmark/alm/gemini/README.md b/examples/benchmark/alm/gemini/README.md index 9b11058..3bc73d8 100644 --- a/examples/benchmark/alm/gemini/README.md +++ b/examples/benchmark/alm/gemini/README.md @@ -22,6 +22,7 @@ audio, so the same service names exist in the llm / vlm / alm leaves. | `gemini-3.5-flash` | `gemini-3.5-flash` | minimal | | `gemini-3.5-flash-lite` | `gemini-3.5-flash-lite` | minimal | | `gemini-3.1-flash-lite` | `gemini-3.1-flash-lite` | minimal | +| `gemini-3.1-pro-preview` | `gemini-3.1-pro-preview` | low (cannot be disabled) | | `gemini-2.5-flash` | `gemini-2.5-flash` | budget 0 | | `gemini-2.5-flash-lite` | `gemini-2.5-flash-lite` | budget 0 | | `gemini-2.5-pro` | `gemini-2.5-pro` | budget 128 (cannot be disabled) | diff --git a/examples/benchmark/alm/gemini/docker-compose.yml b/examples/benchmark/alm/gemini/docker-compose.yml index 553b07b..b933e4f 100644 --- a/examples/benchmark/alm/gemini/docker-compose.yml +++ b/examples/benchmark/alm/gemini/docker-compose.yml @@ -40,6 +40,10 @@ services: <<: *gemini command: > sh -c "python -m _common.run --modality alm --model gemini-3.1-flash-lite --backend gemini --concurrency ${CONCURRENCY:-4} --dataset ${DATASET:-mmau} --limit ${LIMIT:-0} ${EXTRA_ARGS:-}" + gemini-3.1-pro-preview: + <<: *gemini + command: > + sh -c "python -m _common.run --modality alm --model gemini-3.1-pro-preview --backend gemini --concurrency ${CONCURRENCY:-4} --dataset ${DATASET:-mmau} --limit ${LIMIT:-0} ${EXTRA_ARGS:-}" gemini-2.5-flash: <<: *gemini command: > diff --git a/examples/benchmark/llm/gemini/README.md b/examples/benchmark/llm/gemini/README.md index b93da37..e4a5c12 100644 --- a/examples/benchmark/llm/gemini/README.md +++ b/examples/benchmark/llm/gemini/README.md @@ -22,6 +22,7 @@ audio, so the same service names exist in the llm / vlm / alm leaves. | `gemini-3.5-flash` | `gemini-3.5-flash` | minimal | | `gemini-3.5-flash-lite` | `gemini-3.5-flash-lite` | minimal | | `gemini-3.1-flash-lite` | `gemini-3.1-flash-lite` | minimal | +| `gemini-3.1-pro-preview` | `gemini-3.1-pro-preview` | low (cannot be disabled) | | `gemini-2.5-flash` | `gemini-2.5-flash` | budget 0 | | `gemini-2.5-flash-lite` | `gemini-2.5-flash-lite` | budget 0 | | `gemini-2.5-pro` | `gemini-2.5-pro` | budget 128 (cannot be disabled) | diff --git a/examples/benchmark/llm/gemini/docker-compose.yml b/examples/benchmark/llm/gemini/docker-compose.yml index 16e186d..531b8d8 100644 --- a/examples/benchmark/llm/gemini/docker-compose.yml +++ b/examples/benchmark/llm/gemini/docker-compose.yml @@ -40,6 +40,10 @@ services: <<: *gemini command: > sh -c "python -m _common.run --modality llm --model gemini-3.1-flash-lite --backend gemini --concurrency ${CONCURRENCY:-4} --dataset ${DATASET:-bbh_causal_judgement} --limit ${LIMIT:-0} ${EXTRA_ARGS:-}" + gemini-3.1-pro-preview: + <<: *gemini + command: > + sh -c "python -m _common.run --modality llm --model gemini-3.1-pro-preview --backend gemini --concurrency ${CONCURRENCY:-4} --dataset ${DATASET:-bbh_causal_judgement} --limit ${LIMIT:-0} ${EXTRA_ARGS:-}" gemini-2.5-flash: <<: *gemini command: > diff --git a/examples/benchmark/vlm/gemini/README.md b/examples/benchmark/vlm/gemini/README.md index db503e3..5c86d68 100644 --- a/examples/benchmark/vlm/gemini/README.md +++ b/examples/benchmark/vlm/gemini/README.md @@ -22,6 +22,7 @@ audio, so the same service names exist in the llm / vlm / alm leaves. | `gemini-3.5-flash` | `gemini-3.5-flash` | minimal | | `gemini-3.5-flash-lite` | `gemini-3.5-flash-lite` | minimal | | `gemini-3.1-flash-lite` | `gemini-3.1-flash-lite` | minimal | +| `gemini-3.1-pro-preview` | `gemini-3.1-pro-preview` | low (cannot be disabled) | | `gemini-2.5-flash` | `gemini-2.5-flash` | budget 0 | | `gemini-2.5-flash-lite` | `gemini-2.5-flash-lite` | budget 0 | | `gemini-2.5-pro` | `gemini-2.5-pro` | budget 128 (cannot be disabled) | diff --git a/examples/benchmark/vlm/gemini/docker-compose.yml b/examples/benchmark/vlm/gemini/docker-compose.yml index fef8d88..7f9c42b 100644 --- a/examples/benchmark/vlm/gemini/docker-compose.yml +++ b/examples/benchmark/vlm/gemini/docker-compose.yml @@ -40,6 +40,10 @@ services: <<: *gemini command: > sh -c "python -m _common.run --modality vlm --model gemini-3.1-flash-lite --backend gemini --concurrency ${CONCURRENCY:-4} --dataset ${DATASET:-chartqa} --limit ${LIMIT:-0} ${EXTRA_ARGS:-}" + gemini-3.1-pro-preview: + <<: *gemini + command: > + sh -c "python -m _common.run --modality vlm --model gemini-3.1-pro-preview --backend gemini --concurrency ${CONCURRENCY:-4} --dataset ${DATASET:-chartqa} --limit ${LIMIT:-0} ${EXTRA_ARGS:-}" gemini-2.5-flash: <<: *gemini command: > diff --git a/tests/test_examples/test_benchmark_common.py b/tests/test_examples/test_benchmark_common.py index 1a73b88..e85ae04 100644 --- a/tests/test_examples/test_benchmark_common.py +++ b/tests/test_examples/test_benchmark_common.py @@ -26,7 +26,7 @@ def test_every_matrix_cell_resolves_to_a_registered_spec_of_the_right_modality(c models, *_ = common cells = list(models.cells()) - assert len(cells) == 43 # 19 open-weight cells + 8 Gemini models x 3 modalities + assert len(cells) == 46 # 19 open-weight cells + 9 Gemini models x 3 modalities for modality, family, size in cells: resolved = models.resolve(size.key, modality) spec = get_spec(resolved.spec_key) @@ -67,7 +67,7 @@ def test_the_matrix_is_the_one_specified(common): for modality, family, size in models.cells(): by_modality.setdefault(modality, {}).setdefault(family.key, []).append(size.key) gemini = ["gemini-3.7-flash", "gemini-3.6-flash", "gemini-3.5-flash", "gemini-3.5-flash-lite", - "gemini-3.1-flash-lite", "gemini-2.5-flash", "gemini-2.5-flash-lite", "gemini-2.5-pro"] + "gemini-3.1-flash-lite", "gemini-3.1-pro-preview", "gemini-2.5-flash", "gemini-2.5-flash-lite", "gemini-2.5-pro"] assert by_modality == { "vlm": {"qwen": ["qwen3.5-2b", "qwen3.5-4b", "qwen3.5-9b"], "gemma": ["gemma-4-e2b", "gemma-4-e4b", "gemma-4-12b"], @@ -314,7 +314,7 @@ def test_leaf_compose_files_cover_every_cell_and_leave_judge_to_cli_default(comm assert svc["build"]["target"] == family.docker_target and svc["image"] == family.image assert (leaf / "README.md").is_file() and (leaf / ".env").is_symlink() seen.add((modality, family.key, size.key)) - assert len(seen) == 43 + assert len(seen) == 46 def test_dockerfile_has_one_stage_per_family(common): diff --git a/tests/test_models/test_gemini_compat.py b/tests/test_models/test_gemini_compat.py index 7bbb7e9..7f89272 100644 --- a/tests/test_models/test_gemini_compat.py +++ b/tests/test_models/test_gemini_compat.py @@ -210,6 +210,7 @@ def test_thinking_floor_per_model_family(): assert thinking_config("gemini-3.6-flash") == {"thinking_level": "minimal"} assert thinking_config("gemini-3.5-flash-lite") == {"thinking_level": "minimal"} assert thinking_config("gemini-3.1-flash-lite") == {"thinking_level": "minimal"} + assert thinking_config("gemini-3.1-pro-preview") == {"thinking_level": "low"} assert thinking_config("gemini-2.5-flash") == {"thinking_budget": 0} assert thinking_config("gemini-2.5-flash-lite") == {"thinking_budget": 0} assert thinking_config("gemini-2.5-pro") == {"thinking_budget": 128} @@ -223,6 +224,7 @@ def test_explicit_level_is_honoured_but_never_below_the_floor(): assert thinking_config("gemini-3.6-flash", ThinkingPolicy(level="high")) == {"thinking_level": "high"} # 3.7-flash has no minimal: the floor is sent instead assert thinking_config("gemini-3.7-flash", ThinkingPolicy(level="minimal")) == {"thinking_level": "low"} + assert thinking_config("gemini-3.1-pro-preview", ThinkingPolicy(level="minimal")) == {"thinking_level": "low"} # a level asked of a budget model maps to a budget, floored assert thinking_config("gemini-2.5-flash", ThinkingPolicy(level="medium")) == {"thinking_budget": 8192} assert thinking_config("gemini-2.5-pro", ThinkingPolicy(level="minimal")) == {"thinking_budget": 128} @@ -480,7 +482,7 @@ def test_gemini_specs_are_api_only_omni_and_named_by_model_id(): from evalrx.specs import get_spec for key in ("gemini-3.7-flash", "gemini-3.6-flash", "gemini-3.5-flash-lite", - "gemini-3.1-flash-lite", "gemini-2.5-flash-lite"): + "gemini-3.1-flash-lite", "gemini-3.1-pro-preview", "gemini-2.5-flash-lite"): spec = get_spec(key) assert spec.api_only and spec.hf_repo == "" and spec.family == "gemini" assert spec.modalities == frozenset({"text", "image", "audio", "video"})