diff --git a/graphify/llm.py b/graphify/llm.py index 5e410af21..0df9ab410 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -375,6 +375,63 @@ def _resolve_temperature(default: float | None, model: str = "") -> float | None return default +# Same families _THINK_BLOCK_RE strips a / block for: local +# reasoning-tuned models that narrate a long chain of thought before the +# answer. Served through Ollama's OpenAI-compat endpoint, that narration lands +# in a separate `message.reasoning` field (confirmed live against +# nemotron-3-super via raw /v1/chat/completions calls) — `.content` itself is +# never hollow, so this is not a parsing bug. The narration still costs real +# wall-clock and shares the request's token budget, and on a real multi-file +# extraction chunk it can consume enough of both that the client's +# --api-timeout trips, the adaptive bisect-and-retry kicks in, and a chunk can +# still fail at a single file with nothing left to bisect. +_OLLAMA_REASONING_MODEL_FAMILIES = ("nemotron", "deepseek-r1", "qwq") + + +def _ollama_model_is_reasoning(model: str) -> bool: + """True for a local model known to narrate a chain of thought via Ollama.""" + m = (model or "").lower().rsplit("/", 1)[-1] + return any(fam in m for fam in _OLLAMA_REASONING_MODEL_FAMILIES) + + +def _resolve_reasoning_effort(cfg: dict, model: str, backend: str) -> str | None: + """Resolve the `reasoning_effort` value to send, if any. + + Precedence: + 1. GRAPHIFY_OLLAMA_REASONING_EFFORT env var, if `backend == "ollama"` and + it's set — explicit user override. Any backend-recognised value + ("low"/"medium"/"high"), or the literal "none"/"omit"/"default" + (case-insensitive, mirrors GRAPHIFY_LLM_TEMPERATURE's convention) to + force omission even for a recognised reasoning model. + 2. The backend config's own default (only `gemini` sets one today). + 3. For `ollama` with a recognised local reasoning model + (_ollama_model_is_reasoning) and no config default: "high". Measured + empirically against nemotron-3-super (raw API, three otherwise-identical + calls): omitting the field or passing "low" both produced ~17.5k chars + of reasoning and ~5.2k completion tokens; "high" cut that to ~9.2k + chars / ~2.8k tokens — roughly half the time/token budget per call, + which is the difference between finishing under a real extraction + chunk's timeout and not. + 4. Otherwise None (omit the parameter — unchanged default behaviour for + every other backend/model). + + Ollama serves a heterogeneous zoo of models, most of which are not + reasoning-tuned and were never observed with this field, so the default in + step 3 is scoped to named reasoning families rather than applied backend-wide. + """ + if backend == "ollama": + env_raw = os.environ.get("GRAPHIFY_OLLAMA_REASONING_EFFORT", "").strip() + if env_raw: + if env_raw.lower() in ("none", "omit", "default"): + return None + return env_raw + if cfg.get("reasoning_effort"): + return cfg["reasoning_effort"] + if backend == "ollama" and _ollama_model_is_reasoning(model): + return "high" + return None + + def _bedrock_inference_config(max_tokens: int, model: str = "") -> dict: """Build Bedrock inferenceConfig, honouring GRAPHIFY_LLM_TEMPERATURE. @@ -1980,7 +2037,7 @@ def extract_files_direct( mdl, user_msg, temperature=_resolve_temperature(cfg.get("temperature", 0), mdl), - reasoning_effort=cfg.get("reasoning_effort"), + reasoning_effort=_resolve_reasoning_effort(cfg, mdl, backend), # Honour max_completion_tokens (gemini) or the older max_tokens key # (ollama/deepseek/kimi/openai) -- most openai-compat configs define the # latter, so reading only max_completion_tokens silently capped their @@ -2954,8 +3011,9 @@ def _rec(inp, out) -> None: temperature = _resolve_temperature(cfg.get("temperature", 0), mdl) if temperature is not None: kwargs["temperature"] = temperature - if cfg.get("reasoning_effort"): - kwargs["reasoning_effort"] = cfg["reasoning_effort"] + reasoning_effort = _resolve_reasoning_effort(cfg, mdl, backend) + if reasoning_effort: + kwargs["reasoning_effort"] = reasoning_effort # Custom providers can override via providers.json `extra_body`; falls back # to the moonshot default to preserve existing behavior. if cfg.get("extra_body") is not None: diff --git a/tests/test_ollama_reasoning_effort.py b/tests/test_ollama_reasoning_effort.py new file mode 100644 index 000000000..45123f380 --- /dev/null +++ b/tests/test_ollama_reasoning_effort.py @@ -0,0 +1,80 @@ +"""Tests for reasoning_effort resolution on the ollama backend. + +Reasoning-tuned local models served through Ollama (nemotron, deepseek-r1, +qwq — the same families _THINK_BLOCK_RE strips a block for) narrate a +chain of thought in a separate `message.reasoning` field before answering. +`.content` itself is never hollow (confirmed live against nemotron-3-super via +raw /v1/chat/completions calls), so this isn't a JSON-parsing problem — but the +narration shares the request's time/token budget, and on a real multi-file +extraction chunk it can consume enough of both that the client's --api-timeout +trips and a chunk fails with nothing left to bisect. `reasoning_effort="high"` +measurably cuts the narration (and total tokens) roughly in half for the same +prompt, which is the difference between finishing under timeout and not. +""" + +from graphify import llm + + +def test_ollama_model_is_reasoning_matches_known_families(): + assert llm._ollama_model_is_reasoning("nemotron-3-super:120b-a12b-q4_K_M") + assert llm._ollama_model_is_reasoning("deepseek-r1:32b") + assert llm._ollama_model_is_reasoning("qwq:32b") + # Provider-prefixed form, matching _model_requires_default_temperature's + # existing "openai/gpt-5" style stripping. + assert llm._ollama_model_is_reasoning("nvidia/nemotron-3-super") + + +def test_ollama_model_is_reasoning_false_for_unrecognised_models(): + assert not llm._ollama_model_is_reasoning("qwen2.5-coder:7b") + assert not llm._ollama_model_is_reasoning("llama3.1:8b") + assert not llm._ollama_model_is_reasoning("") + assert not llm._ollama_model_is_reasoning(None) + + +def test_resolve_reasoning_effort_defaults_high_for_ollama_reasoning_model(monkeypatch): + monkeypatch.delenv("GRAPHIFY_OLLAMA_REASONING_EFFORT", raising=False) + + assert llm._resolve_reasoning_effort({}, "nemotron-3-super:120b", "ollama") == "high" + + +def test_resolve_reasoning_effort_none_for_ollama_non_reasoning_model(monkeypatch): + monkeypatch.delenv("GRAPHIFY_OLLAMA_REASONING_EFFORT", raising=False) + + assert llm._resolve_reasoning_effort({}, "qwen2.5-coder:7b", "ollama") is None + + +def test_resolve_reasoning_effort_unaffected_for_non_ollama_backends(monkeypatch): + monkeypatch.setenv("GRAPHIFY_OLLAMA_REASONING_EFFORT", "high") + + # Named "nemotron" or not, a non-ollama backend never gets the ollama-only + # default or the ollama-only env override. + assert llm._resolve_reasoning_effort({}, "nemotron-3-super:120b", "claude") is None + assert llm._resolve_reasoning_effort( + {"reasoning_effort": "low"}, "gemini-3-flash-preview", "gemini" + ) == "low" + + +def test_resolve_reasoning_effort_env_override_wins_over_default(monkeypatch): + monkeypatch.setenv("GRAPHIFY_OLLAMA_REASONING_EFFORT", "low") + + assert llm._resolve_reasoning_effort({}, "nemotron-3-super:120b", "ollama") == "low" + # Env override applies even to a model that wouldn't otherwise default to + # anything — an explicit user choice always wins. + assert llm._resolve_reasoning_effort({}, "qwen2.5-coder:7b", "ollama") == "low" + + +def test_resolve_reasoning_effort_env_force_omit(monkeypatch): + for sentinel in ("none", "omit", "default", "NONE", "Omit"): + monkeypatch.setenv("GRAPHIFY_OLLAMA_REASONING_EFFORT", sentinel) + assert llm._resolve_reasoning_effort({}, "nemotron-3-super:120b", "ollama") is None + + +def test_resolve_reasoning_effort_backend_config_default_takes_precedence(monkeypatch): + monkeypatch.delenv("GRAPHIFY_OLLAMA_REASONING_EFFORT", raising=False) + + # If a future ollama-family config ever sets its own default, that wins + # over the reasoning-model fallback rather than being silently overridden. + assert ( + llm._resolve_reasoning_effort({"reasoning_effort": "medium"}, "nemotron-3-super:120b", "ollama") + == "medium" + )