Make the model a config string: LiteLLM replaces the Ollama SDK

backend/app/sentinel_agent/agent.py

@@ -31,7 +31,15 @@
 import logging
 from typing import Any
 
-from app.sentinel_agent.llm import LLMProvider
+from app.sentinel_agent.llm import (
+    LLMProvider,
+    assistant_message,
+    has_images,
+    image_message,
+    prune_images,
+    tool_call_arguments,
+    tool_result_message,
+)
 from app.sentinel_agent.mcp_client import MCPClientManager
 from app.sentinel_agent.prompts import initial_user_message, system_prompt_for_trigger
 
@@ -110,7 +118,7 @@ async def run(self, run: dict) -> dict[str, Any]:
                     incident_id, severity, f"LLM call failed: {exc}", tool_trace,
                 )
 
-            messages.append(response_msg)
+            messages.append(assistant_message(response_msg))
 
             # Terminal: model responded without calling a tool.
             if not response_msg.tool_calls:
@@ -144,11 +152,12 @@ async def run(self, run: dict) -> dict[str, Any]:
             # any of it yet).
             batch_start = len(messages)
             for tc in response_msg.tool_calls:
-                args = (
-                    tc.function.arguments
-                    if isinstance(tc.function.arguments, dict)
-                    else {}
-                )
+                # OpenAI-shaped providers hand arguments back as a JSON
+                # string, Ollama as a dict; the helper normalises both and
+                # degrades to {} on malformed JSON rather than killing the
+                # run — the tool then fails its own validation with a
+                # message the model can react to.
+                args = tool_call_arguments(tc)
                 logger.info("agent: tool %s(%s)", tc.function.name, args)
 
                 result = await self.mcp.call_tool(tc.function.name, args)
@@ -184,17 +193,15 @@ async def run(self, run: dict) -> dict[str, Any]:
                         incident_id = parsed_id
 
                 # Feed the tool result back to the LLM.
-                messages.append({
-                    "role": "tool",
-                    "tool_name": tc.function.name,
-                    "content": result.get("text", ""),
-                })
+                messages.append(
+                    tool_result_message(tc, result.get("text", ""))
+                )
                 if result.get("images"):
-                    messages.append({
-                        "role": "user",
-                        "content": f"[Visual output from {tc.function.name}]",
-                        "images": result["images"],
-                    })
+                    # Separate user message: an OpenAI-shaped `tool`
+                    # message cannot carry an image at all.
+                    messages.append(
+                        image_message(tc.function.name, result["images"])
+                    )
 
             # Image pruning: every base64 frame appended above is
             # otherwise RE-SENT on every subsequent LLM call — O(N²)
@@ -208,13 +215,9 @@ async def run(self, run: dict) -> dict[str, Any]:
             # and pruning to "newest only" here would make the model
             # assess cameras whose frames it never saw.
             for stale in messages[:batch_start]:
-                if not (isinstance(stale, dict) and stale.get("images")):
+                if not has_images(stale):
                     continue
-                stale.pop("images", None)
-                stale["content"] = (
-                    f"{stale.get('content', '')} [frames pruned — superseded "
-                    "by newer visual output]"
-                )
+                prune_images(stale)
 
         # Iteration budget exhausted.
         return _truncated_result(

backend/app/sentinel_agent/config.py

@@ -17,8 +17,32 @@
 
 
 class Settings(BaseSettings):
-    # ── LLM (Ollama Cloud) ───────────────────────────────────────────
-    ollama_api_key: str
+    # ── LLM (via LiteLLM) ────────────────────────────────────────────
+    # LLM_MODEL is a LiteLLM model string and is the ONLY setting needed
+    # to change provider:
+    #
+    #   ollama_chat/qwen3.5:cloud        Ollama Cloud (today's default)
+    #   anthropic/claude-sonnet-5        bring-your-own-key
+    #   openai/gpt-5.1
+    #   openai/sentinel-1                a future SourceBox model behind
+    #                                    an OpenAI-compatible endpoint
+    #
+    # Leave it unset and the OLLAMA_* settings below are used instead —
+    # which is what keeps every existing deployment working untouched.
+    #
+    # Whatever you point this at MUST support tool calling and image
+    # input. The agent loop is built on both: it fans out tool calls and
+    # feeds camera frames back in. A text-only model does not degrade
+    # here, it fails every run.
+    llm_model: str = ""
+    llm_api_key: str = ""
+    llm_api_base: str = ""
+
+    # ── LLM (Ollama Cloud) — the default provider ────────────────────
+    # No longer required: a deployment can configure LLM_* instead. The
+    # validator below still refuses to boot with no credential at all,
+    # which is what this field's missing-by-default used to enforce.
+    ollama_api_key: str = ""
     ollama_host: str = "https://ollama.com"
     # Vision-capable + tool-calling.  qwen3.5:cloud has 256K context,
     # which matters for the multi-turn tool-use loop where the
@@ -147,6 +171,54 @@ class Settings(BaseSettings):
     # someone left an old setting around.
     model_config = {"env_file": ".env", "extra": "ignore"}
 
+    # ── Resolved LLM settings ────────────────────────────────────────
+    # Explicit LLM_* wins; otherwise fall back to the OLLAMA_* trio so an
+    # existing deployment keeps working with no env changes at all. This
+    # is the whole backward-compatibility story for the LiteLLM move.
+
+    @property
+    def resolved_llm_model(self) -> str:
+        if self.llm_model:
+            return self.llm_model
+        # `ollama_chat/` (not `ollama/`) — the chat endpoint is the one
+        # that supports tool calling, which the agent loop requires.
+        return f"ollama_chat/{self.ollama_model}"
+
+    @property
+    def resolved_llm_api_key(self) -> str:
+        return self.llm_api_key or self.ollama_api_key
+
+    @property
+    def resolved_llm_api_base(self) -> str:
+        if self.llm_api_base:
+            return self.llm_api_base
+        # Only meaningful for the Ollama fallback; a hosted provider
+        # resolves its own endpoint from the model string.
+        return self.ollama_host if not self.llm_model else ""
+
+    @model_validator(mode="after")
+    def _require_an_llm_credential(self):
+        """Fail at boot, not mid-run, when no credential is configured.
+
+        `ollama_api_key` used to be a required field, so a missing key
+        was a startup ValidationError. Making it optional (so LLM_* can
+        be used instead) would otherwise have turned that into a run
+        that fetches work, calls the model, and errors — burning a
+        wakeup and stranding the run.
+
+        Skipped when api_base points somewhere local: a self-hosted
+        Ollama on the same box legitimately needs no key.
+        """
+        if self.resolved_llm_api_key:
+            return self
+        base = self.resolved_llm_api_base
+        if any(h in base for h in ("localhost", "127.0.0.1", "host.docker.internal")):
+            return self
+        raise ValueError(
+            "No LLM credential configured — set LLM_API_KEY (or "
+            "OLLAMA_API_KEY). The agent cannot run without one."
+        )
+
     @model_validator(mode="after")
     def _default_mcp_key_to_agent_key(self):
         """Let one key configure a self-hosted agent.

backend/app/sentinel_agent/llm.py

@@ -1,20 +1,53 @@
+"""LLM access for the agent, via LiteLLM.
+
+One provider path for every model. LiteLLM speaks Ollama natively
+(``ollama_chat/<model>``), so today's hosted model, a self-hoster's
+bring-your-own-key provider, and eventually SourceBox's own model are all
+selected by a single ``LLM_MODEL`` string rather than by swapping client
+libraries.
+
+This module owns ALL provider-shaped knowledge. The agent loop builds
+messages exclusively through the helpers below, so if the message format
+ever changes again it changes here and nowhere else. That mattered when
+this moved off the Ollama SDK: Ollama and OpenAI-shaped APIs disagree
+about more than field names.
+
+  - Tool results: Ollama keys them by ``tool_name``, OpenAI by
+    ``tool_call_id`` matched to the id it issued on the call.
+  - Tool arguments: Ollama hands back a dict, OpenAI a JSON *string*.
+  - Images: Ollama takes raw base64 in an ``images`` list on any message.
+    OpenAI-shaped APIs cannot carry an image on a ``tool`` message at
+    all — it has to be a ``user`` message whose content is a list of
+    parts, with the image as a ``data:`` URI. That is a structural
+    difference, not a formatting one, and it is why the pruning helper
+    below has to rewrite a content list rather than pop a key.
+"""
+
 import asyncio
+import json
 import logging
+from typing import Any
 
-from ollama import AsyncClient
+import litellm
 
 from app.sentinel_agent.config import Settings
 
 logger = logging.getLogger(__name__)
 
+# LiteLLM chats to stdout about provider quirks on import and on first
+# call; the agent's logs are read during incidents and this is noise.
+litellm.suppress_debug_info = True
+
+# Marker left on a message whose frames were pruned, so the pruning pass
+# is idempotent and a re-prune doesn't stack notices.
+_PRUNED_NOTE = "[frames pruned — superseded by newer visual output]"
+
 
 class LLMProvider:
     def __init__(self, settings: Settings) -> None:
-        self.client = AsyncClient(
-            host=settings.ollama_host,
-            headers={"Authorization": f"Bearer {settings.ollama_api_key}"},
-        )
-        self.model = settings.ollama_model
+        self.model = settings.resolved_llm_model
+        self.api_key = settings.resolved_llm_api_key
+        self.api_base = settings.resolved_llm_api_base
         self.max_tokens = settings.max_tokens
         self.timeout_seconds = settings.llm_call_timeout_seconds
 
@@ -23,20 +56,133 @@ async def chat(
         messages: list[dict],
         tools: list[dict] | None = None,
     ):
-        """Send a chat request and return the response message.
+        """Send a chat request and return the assistant message.
 
-        Wrapped in asyncio.wait_for because the Ollama AsyncClient does
-        not honour a per-call timeout and a hung connection would
-        otherwise block until the wakeup's 270 s wall clock fires —
-        wasting Fly minutes and stranding the in-flight run.
+        Still wrapped in ``asyncio.wait_for`` despite LiteLLM taking its
+        own ``timeout``: that one bounds the HTTP request, not the whole
+        call, so a provider that accepts the connection and then stalls
+        mid-stream would otherwise hold the machine alive until the
+        270 s wall clock in process_with_timeout fires — burning Fly
+        minutes and stranding the run in `running` on Command Center.
         """
+        kwargs: dict[str, Any] = {
+            "model": self.model,
+            "messages": messages,
+            "max_tokens": self.max_tokens,
+            "timeout": self.timeout_seconds,
+        }
+        if tools:
+            kwargs["tools"] = tools
+        if self.api_key:
+            kwargs["api_key"] = self.api_key
+        if self.api_base:
+            kwargs["api_base"] = self.api_base
+
         response = await asyncio.wait_for(
-            self.client.chat(
-                model=self.model,
-                messages=messages,
-                tools=tools or None,
-                options={"num_predict": self.max_tokens},
-            ),
+            litellm.acompletion(**kwargs),
             timeout=self.timeout_seconds,
         )
-        return response.message
+        return response.choices[0].message
+
+
+# ── Message construction (the only place the wire format is known) ────
+
+
+def assistant_message(response_msg) -> dict:
+    """Convert the provider's assistant message into a plain dict.
+
+    Appended verbatim to the running `messages` list, so it has to be
+    JSON-serialisable — a provider object round-trips inconsistently
+    once it goes back over the wire.
+    """
+    if hasattr(response_msg, "model_dump"):
+        msg = response_msg.model_dump()
+    else:  # pragma: no cover - defensive
+        msg = dict(response_msg)
+    # Drop null-valued keys some providers include; a `tool_calls: null`
+    # is rejected by others on the next request.
+    return {k: v for k, v in msg.items() if v is not None}
+
+
+def tool_call_arguments(tool_call) -> dict:
+    """Arguments for a tool call, always as a dict.
+
+    OpenAI-shaped APIs return a JSON *string* here. A model can emit
+    malformed JSON, and that must not take the whole run down — an empty
+    dict lets the tool fail on its own validation with a message the
+    model can actually react to.
+    """
+    raw = tool_call.function.arguments
+    if isinstance(raw, dict):
+        return raw
+    if not raw:
+        return {}
+    try:
+        parsed = json.loads(raw)
+    except (TypeError, ValueError):
+        logger.warning(
+            "llm: tool %s had unparseable arguments: %.200r",
+            tool_call.function.name, raw,
+        )
+        return {}
+    return parsed if isinstance(parsed, dict) else {}
+
+
+def tool_result_message(tool_call, text: str) -> dict:
+    """The tool's textual result, keyed back to the call that asked."""
+    return {
+        "role": "tool",
+        "tool_call_id": tool_call.id,
+        "name": tool_call.function.name,
+        "content": text or "",
+    }
+
+
+def image_message(tool_name: str, images: list[str]) -> dict:
+    """Frames from a tool, as a user message with image parts.
+
+    A separate message because an OpenAI-shaped ``tool`` message cannot
+    carry an image. ``images`` are raw base64 (no prefix) as returned by
+    the MCP layer; the ``data:`` URI is applied here so the MCP client
+    stays provider-neutral.
+    """
+    parts: list[dict] = [
+        {"type": "text", "text": f"[Visual output from {tool_name}]"}
+    ]
+    for b64 in images:
+        parts.append({
+            "type": "image_url",
+            "image_url": {"url": f"data:image/jpeg;base64,{b64}"},
+        })
+    return {"role": "user", "content": parts}
+
+
+def has_images(message: Any) -> bool:
+    if not isinstance(message, dict):
+        return False
+    content = message.get("content")
+    if not isinstance(content, list):
+        return False
+    return any(
+        isinstance(p, dict) and p.get("type") == "image_url" for p in content
+    )
+
+
+def prune_images(message: dict) -> None:
+    """Strip image parts, collapsing the message back to its text.
+
+    Mutates in place, matching how the caller walks the message list.
+    Idempotent — a message already pruned is left alone.
+    """
+    content = message.get("content")
+    if not isinstance(content, list):
+        return
+    texts = [
+        p.get("text", "") for p in content
+        if isinstance(p, dict) and p.get("type") == "text"
+    ]
+    joined = " ".join(t for t in texts if t).strip()
+    if _PRUNED_NOTE in joined:
+        message["content"] = joined
+        return
+    message["content"] = f"{joined} {_PRUNED_NOTE}".strip()

backend/app/sentinel_agent/mcp_client.py

@@ -128,7 +128,11 @@ def get_tools_for_llm(self) -> list[dict]:
     async def call_tool(self, tool_name: str, arguments: dict) -> dict:
         """Execute a tool and return {text: str, images: list[str]}.
 
-        images are raw base64 strings (no data URI prefix) for Ollama.
+        images are raw base64 strings (no data URI prefix). Kept
+        provider-neutral on purpose: llm.image_message() applies whatever
+        wrapping the configured model wants (currently a `data:` URI in
+        an OpenAI-style image_url part), so switching providers never
+        reaches back into the MCP layer.
         """
         if tool_name not in self._tools:
             return {