From 7cebb3c2e89075adaab928799b7851b08b642aab Mon Sep 17 00:00:00 2001 From: Zhang Jiahao Date: Sun, 8 Nov 2026 19:20:13 -0600 Subject: [PATCH 1/2] mini-swe-agent: a Base Agent, an acting agent and a deployment host - `mini-swe-agent` is a CLI Base Agent beside Codex, Claude Code and Pi (`MiniSweAgent` in CLIS): one call runs one task through mini-swe-agent's Python API in a process of its own (hermit/agent/base/mini_swe_agent_run.py), which writes the session as JSON events, the stream CliAgent keeps and reads into the reply, the trajectory steps and the usage. Isolation, the home, the tool server, the clock and the retries of a provider that could not serve are CliAgent's. The call's tools are one bash command, `hermit-tool`, answered through the tool server's JSON entrance; the model's key goes with each request, the commands get only `env`, and on Linux the process is non-dumpable. A setting that fails every task (a config that names the model, a wrong key or model id found by the first call) starts no session; at the timeout the running command is killed too. - `agent.name: mini-swe-agent` (or `agent.base: mini-swe-agent`) is the workspace Acting Agent with that backend, its model bound as the other CLIs' are; a reference takes the litellm model classes only. - `hermit deploy --host mini-swe-agent` deploys as every host does (the copy under hermit//, what mini-swe-agent cannot load named in the context) and writes `mini-swe-agent.yaml`, whose system template gives `mini` the system message the Base Agent sends. - The `mini-swe-agent` extra (installed by CI), a tutorial and an example, and the agent and host in the reference tables. --- .github/workflows/tests.yml | 5 +- CHANGELOG.md | 6 + .../mini_swe_agent/environment/inventory.csv | 13 + docs/_examples/mini_swe_agent/judge.py | 31 ++ docs/_examples/mini_swe_agent/run.yaml | 37 ++ .../mini_swe_agent/task-info/README.md | 8 + docs/_examples/mini_swe_agent/tasks.jsonl | 9 + docs/_examples/mini_swe_agent/test.txt | 3 + docs/_examples/mini_swe_agent/train.txt | 3 + docs/_examples/mini_swe_agent/val.txt | 3 + docs/faq.md | 2 +- docs/framework/adapter.md | 28 +- docs/framework/agent-layers.md | 17 +- docs/framework/commands.md | 6 +- docs/framework/configuration.md | 4 +- docs/framework/task.md | 3 +- docs/framework/what-is-a-harness-adapter.md | 2 +- docs/get-started/installation.md | 11 + docs/guides/model-backends.md | 1 + docs/index.md | 1 + docs/reference/compatibility.md | 4 +- docs/reference/config-fields.md | 8 +- docs/tutorials/mini-swe-agent.md | 126 ++++++ docs/web/index.html | 2 +- hermit/agent/acting/__init__.py | 2 +- hermit/agent/acting/agent.py | 9 +- hermit/agent/acting/workspace.py | 33 +- hermit/agent/base/cli.py | 163 ++++++- hermit/agent/base/mini_swe_agent.py | 142 ++++++ hermit/agent/base/mini_swe_agent.yaml | 210 +++++++++ hermit/agent/base/mini_swe_agent_env.py | 168 +++++++ hermit/agent/base/mini_swe_agent_run.py | 275 ++++++++++++ hermit/agent/base/records.py | 24 +- hermit/agent/base/trajectory.py | 54 ++- hermit/agent/model/backends.py | 27 +- hermit/cli.py | 7 +- hermit/config.py | 15 +- hermit/defaults.yaml | 7 +- hermit/hosts/mini_swe_agent.py | 100 +++++ hermit/hosts/mini_swe_agent_env.py | 50 +++ hermit/isolate.py | 4 +- pyproject.toml | 5 +- tests/core/test_context_tool.py | 4 +- tests/core/test_mini_swe_agent.py | 418 ++++++++++++++++++ 44 files changed, 1965 insertions(+), 85 deletions(-) create mode 100644 docs/_examples/mini_swe_agent/environment/inventory.csv create mode 100644 docs/_examples/mini_swe_agent/judge.py create mode 100644 docs/_examples/mini_swe_agent/run.yaml create mode 100644 docs/_examples/mini_swe_agent/task-info/README.md create mode 100644 docs/_examples/mini_swe_agent/tasks.jsonl create mode 100644 docs/_examples/mini_swe_agent/test.txt create mode 100644 docs/_examples/mini_swe_agent/train.txt create mode 100644 docs/_examples/mini_swe_agent/val.txt create mode 100644 docs/tutorials/mini-swe-agent.md create mode 100644 hermit/agent/base/mini_swe_agent.py create mode 100644 hermit/agent/base/mini_swe_agent.yaml create mode 100644 hermit/agent/base/mini_swe_agent_env.py create mode 100644 hermit/agent/base/mini_swe_agent_run.py create mode 100644 hermit/hosts/mini_swe_agent.py create mode 100644 hermit/hosts/mini_swe_agent_env.py create mode 100644 tests/core/test_mini_swe_agent.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 38834366..135deb5b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,8 +34,9 @@ jobs: sudo apt-get install -y bubblewrap strace python -m venv "$RUNNER_TEMP/test-env" "$RUNNER_TEMP/test-env/bin/python" -m pip install --upgrade pip - # the methods' dependency groups, so their tests run rather than skip (not ace's: it pulls torch) - "$RUNNER_TEMP/test-env/bin/python" -m pip install -e '.[test]' \ + # the methods' dependency groups and the mini-swe-agent extra, so their tests run rather than skip (not + # ace's group: it pulls torch) + "$RUNNER_TEMP/test-env/bin/python" -m pip install -e '.[test,mini-swe-agent]' \ --group gepa --group skillopt --group harness2 --group rrsi echo "$RUNNER_TEMP/test-env/bin" >> "$GITHUB_PATH" - name: The test groups this pull request touches diff --git a/CHANGELOG.md b/CHANGELOG.md index 335235c3..7fc718ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,12 @@ Changes are grouped by software version. Entries under Unreleased describe work call's tools are an MCP server offered directly; a failed request fails the session although Pi exits 0. Pi takes no roles and has no web search; `image_input: true` declares a bound model that takes images. `hermit deploy --host pi` declares the adapter's tools `exposure: direct`, with a 120 s timeout. +- mini-swe-agent as a CLI Base Agent (`mini-swe-agent`): the acting agent (`agent.name` or `agent.base: + mini-swe-agent`) runs one task through mini-swe-agent 2.4.6's Python API in a process of its own, whose JSON + events are the stream; a model reference is called through litellm with the key passed per request, and the + agent's commands get only `env`. The call's tools are one bash command, `hermit-tool`. `hermit deploy --host + mini-swe-agent` writes `mini-swe-agent.yaml`. Installed with the `mini-swe-agent` extra. + [Tutorial](docs/tutorials/mini-swe-agent.md). - Models in sandboxes as Harbor's agents get theirs: each sandbox gets its models' key variables and provider hosts (an isolated network becomes an allowlist) and forwards a model meter on the loopback; what a run keeps (the learning material, the step records, the adapter) is scrubbed of key values (`hermit/redact.py`). The allowlist diff --git a/docs/_examples/mini_swe_agent/environment/inventory.csv b/docs/_examples/mini_swe_agent/environment/inventory.csv new file mode 100644 index 00000000..fcbf24e9 --- /dev/null +++ b/docs/_examples/mini_swe_agent/environment/inventory.csv @@ -0,0 +1,13 @@ +item,warehouse,units +bolt,north,120 +bolt,south,75 +nut,north,300 +nut,east,45 +washer,south,80 +washer,east,20 +bracket,north,12 +bracket,south,30 +hinge,east,64 +hinge,north,16 +spring,south,9 +spring,east,41 diff --git a/docs/_examples/mini_swe_agent/judge.py b/docs/_examples/mini_swe_agent/judge.py new file mode 100644 index 00000000..07729b85 --- /dev/null +++ b/docs/_examples/mini_swe_agent/judge.py @@ -0,0 +1,31 @@ +"""Private answers are kept in the judge, outside learner-visible task rows.""" +import json + +from hermit.task import JudgeResult, RunOutput + +ANSWERS = { + "train-001": 195, + "train-002": 170, + "train-003": 4, + "val-001": 345, + "val-002": 194, + "val-003": 6, + "test-001": 80, + "test-002": 448, + "test-003": 5, +} + + +def score(task: dict, output: RunOutput) -> JudgeResult: + expected = ANSWERS[task["id"]] + try: + value = json.loads(output.output) + except (ValueError, TypeError): + value = None + correct = isinstance(value, dict) and value == {"answer": expected} + feedback = "Correct." if correct else ( + 'Return only a JSON object with exactly one key, "answer", whose value is an integer, for example ' + '{"answer": 12}. No Markdown, units or explanation. ' + f"The reference answer for this attempt is {expected}." + ) + return JudgeResult(score=float(correct), feedback=feedback) diff --git a/docs/_examples/mini_swe_agent/run.yaml b/docs/_examples/mini_swe_agent/run.yaml new file mode 100644 index 00000000..fd9f27d5 --- /dev/null +++ b/docs/_examples/mini_swe_agent/run.yaml @@ -0,0 +1,37 @@ +# mini-swe-agent as the acting agent: nine questions about a CSV file, answered with bash commands. See +# docs/tutorials/mini-swe-agent.md. One model, a `/` reference (hermit backends), acts and learns. +model: deepseek/deepseek-flash +data: + tasks: tasks.jsonl + train: train.txt + val: val.txt + test: test.txt + step_size: 3 + shuffle: false +task_info: task-info +judge: + path: judge.py +template: prompt-only@1.0 +evolver: + name: gepa + params: + max_tokens: 4096 +sandbox: + enforce: best_effort # bubblewrap where it is installed; elsewhere unconfined, recorded and warned +environment: + read: [environment] # the task's file, copied into each workspace; the data, the judge and the runs stay out +trainer: + run_name: mini-swe-agent + output_dir: runs + epochs: 1 + budget: + tokens: 600000 + hours: 0.5 + call_timeout_min: 10 +agent: + name: mini-swe-agent + concurrency: 3 + params: + files: [environment/inventory.csv] + web_search: false # set either way for every CLI agent; mini-swe-agent has none + timeout_s: 300 diff --git a/docs/_examples/mini_swe_agent/task-info/README.md b/docs/_examples/mini_swe_agent/task-info/README.md new file mode 100644 index 00000000..1a11cbf5 --- /dev/null +++ b/docs/_examples/mini_swe_agent/task-info/README.md @@ -0,0 +1,8 @@ +# Inventory questions + +Each task asks one question about `inventory.csv`, a small table of items, warehouses and unit counts that is copied +into the agent's working directory. The agent answers by running bash commands, mini-swe-agent's only action. + +The judge accepts exactly `{"answer": }`; after a wrong or malformed answer its feedback gives the format +rule and the reference answer. Learn the output format from training feedback. Test tasks are held out from the +learner. diff --git a/docs/_examples/mini_swe_agent/tasks.jsonl b/docs/_examples/mini_swe_agent/tasks.jsonl new file mode 100644 index 00000000..5f0373f5 --- /dev/null +++ b/docs/_examples/mini_swe_agent/tasks.jsonl @@ -0,0 +1,9 @@ +{"id":"train-001","prompt":"How many units of bolt are there across all warehouses in inventory.csv?"} +{"id":"train-002","prompt":"How many units are stored in the east warehouse in inventory.csv?"} +{"id":"train-003","prompt":"How many rows of inventory.csv have fewer than 25 units?"} +{"id":"val-001","prompt":"How many units of nut are there across all warehouses in inventory.csv?"} +{"id":"val-002","prompt":"How many units are stored in the south warehouse in inventory.csv?"} +{"id":"val-003","prompt":"How many distinct items does inventory.csv list?"} +{"id":"test-001","prompt":"How many units of hinge are there across all warehouses in inventory.csv?"} +{"id":"test-002","prompt":"How many units are stored in the north warehouse in inventory.csv?"} +{"id":"test-003","prompt":"How many rows of inventory.csv have more than 60 units?"} diff --git a/docs/_examples/mini_swe_agent/test.txt b/docs/_examples/mini_swe_agent/test.txt new file mode 100644 index 00000000..42cca025 --- /dev/null +++ b/docs/_examples/mini_swe_agent/test.txt @@ -0,0 +1,3 @@ +test-001 +test-002 +test-003 diff --git a/docs/_examples/mini_swe_agent/train.txt b/docs/_examples/mini_swe_agent/train.txt new file mode 100644 index 00000000..417d9a47 --- /dev/null +++ b/docs/_examples/mini_swe_agent/train.txt @@ -0,0 +1,3 @@ +train-001 +train-002 +train-003 diff --git a/docs/_examples/mini_swe_agent/val.txt b/docs/_examples/mini_swe_agent/val.txt new file mode 100644 index 00000000..3004e27f --- /dev/null +++ b/docs/_examples/mini_swe_agent/val.txt @@ -0,0 +1,3 @@ +val-001 +val-002 +val-003 diff --git a/docs/faq.md b/docs/faq.md index 63cf9ac7..75fed021 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -61,7 +61,7 @@ Follow [Deploy an adapter](tutorials/deploy-an-adapter.md) for the commands and
Can I Reuse an Adapter Across Models and Harnesses? -Yes, when the receiving agent supports the exports the adapter needs. Hermit currently generates workspaces for Codex, Claude Code, and Pi, and converts native plugin packages for Claude Code, Codex, Pi, and Hermes. Each route has different capabilities; a target may refuse unsupported exports. +Yes, when the receiving agent supports the exports the adapter needs. Hermit currently generates workspaces for Codex, Claude Code, Pi, and mini-swe-agent, and converts native plugin packages for Claude Code, Codex, Pi, and Hermes. Each route has different capabilities; a target may refuse unsupported exports. Compatibility does not guarantee the same performance on a different model, harness, or task. Evaluate the adapter in its new setting. See the [host capability matrix](framework/commands.md#what-each-hosts-plugin-carries) and [compatibility limits](reference/compatibility.md). diff --git a/docs/framework/adapter.md b/docs/framework/adapter.md index 41062547..f619eda3 100644 --- a/docs/framework/adapter.md +++ b/docs/framework/adapter.md @@ -76,18 +76,18 @@ id>/`. Paths in the instructions below, and in the adapter's skills and roles, a command they give from there." Then each export `exports.yaml` declares is connected to the host's own place when the host has one, so that the host loads it; an export the host has no place for stays a file of the copy, and the context names it with what to do with it. The rule is the same for every -host, and for the built-in `codex` and `claude-code` acting agents, which deploy as `hermit deploy` does: an adapter +host, and for the built-in `codex`, `claude-code`, `pi` and `mini-swe-agent` acting agents, which deploy as `hermit deploy` does: an adapter is deployed as far as the host can take it, and nothing it holds is out of the agent's reach. -| Adapter source | Deployment operation | Codex output | Claude Code output | Pi output | -|---|---|---|---|---| -| Every file but `.hermit/` and `.git/` | Copy it as it is. | `hermit//` | `hermit//` | `hermit//` | -| `ADAPTER.md` (always exported) | Insert the line above and its text into a managed instruction block. | `AGENTS.md` | `CLAUDE.md` | `AGENTS.md` | -| `skills/verify-output/`, through `skills` in `exports.yaml` | Copy the skill directory. | `.codex/skills/verify-output/` | `.claude/skills/verify-output/` | `.pi/skills/verify-output/` | -| `agents/reviewer.md` (or `agents/reviewer/AGENT.md`), through `agents` in `exports.yaml` | Convert the subagent into native host files: frontmatter gives name and description, the body becomes its instructions. | `.codex/agents/reviewer.toml`, registered in `.codex/config.toml` | `.claude/agents/reviewer.md` | named in `AGENTS.md` (Pi has no subagents): name, description, the path of its instructions | -| `tools.py`, through `tools` in `exports.yaml` | Configure a framework MCP server that loads the source adapter. | `.codex/config.toml` | `.mcp.json` | `.pi/mcp.json` | -| a hooks file, through `hooks` in `exports.yaml` | Write the hooks the host runs, each command run from `hermit//`; name the others in the context (a command to run, a `prompt` hook's check to make at that moment); a hook of another type is refused. | `.codex/hooks.json` (command hooks; Codex runs no prompt or agent hooks) | `hooks` in `.claude/settings.json` (command and prompt hooks) | named in `AGENTS.md` (Pi reads no hook file) | -| every other file of the copy | Name it, with its frontmatter description or first line, under "Adapter resources" in the context (up to 60 files; the rest are counted). | `AGENTS.md` | `CLAUDE.md` | `AGENTS.md` | +| Adapter source | Deployment operation | Codex output | Claude Code output | Pi output | mini-swe-agent output | +|---|---|---|---|---|---| +| Every file but `.hermit/` and `.git/` | Copy it as it is. | `hermit//` | `hermit//` | `hermit//` | `hermit//` | +| `ADAPTER.md` (always exported) | Insert the line above and its text into a managed instruction block. | `AGENTS.md` | `CLAUDE.md` | `AGENTS.md` | the system template of `mini-swe-agent.yaml` | +| `skills/verify-output/`, through `skills` in `exports.yaml` | Copy the skill directory. | `.codex/skills/verify-output/` | `.claude/skills/verify-output/` | `.pi/skills/verify-output/` | named in the system template (mini-swe-agent loads no skills): name, description, the path of its `SKILL.md` | +| `agents/reviewer.md` (or `agents/reviewer/AGENT.md`), through `agents` in `exports.yaml` | Convert the subagent into native host files: frontmatter gives name and description, the body becomes its instructions. | `.codex/agents/reviewer.toml`, registered in `.codex/config.toml` | `.claude/agents/reviewer.md` | named in `AGENTS.md` (Pi has no subagents): name, description, the path of its instructions | named in the system template (no subagents) | +| `tools.py`, through `tools` in `exports.yaml` | Configure a framework MCP server that loads the source adapter. | `.codex/config.toml` | `.mcp.json` | `.pi/mcp.json` | `environment` in `mini-swe-agent.yaml`: Hermit's environment class answers the `hermit-tool` command | +| a hooks file, through `hooks` in `exports.yaml` | Write the hooks the host runs, each command run from `hermit//`; name the others in the context (a command to run, a `prompt` hook's check to make at that moment); a hook of another type is refused. | `.codex/hooks.json` (command hooks; Codex runs no prompt or agent hooks) | `hooks` in `.claude/settings.json` (command and prompt hooks) | named in `AGENTS.md` (Pi reads no hook file) | named in the system template (mini-swe-agent runs no hooks) | +| every other file of the copy | Name it, with its frontmatter description or first line, under "Adapter resources" in the context (up to 60 files; the rest are counted). | `AGENTS.md` | `CLAUDE.md` | `AGENTS.md` | the system template | This is how the adapter reaches the acting system. The host reads the generated instruction, skill, role and hook files; tool configuration makes the declared tools available through its tool interface. What the adapter does not @@ -140,6 +140,14 @@ pi-workspace/ # only for an adapter without roles .hermit-deploy.json ``` +```text +mini-swe-agent-workspace/ + mini-swe-agent.yaml # the context, the resources and the tools in its system template + hermit/file-example/ + ADAPTER.md exports.yaml tools.py skills/ agents/ README.md + .hermit-deploy.json +``` + Only declared exports are connected; every file is copied. A prompt-only adapter has no tool-server entry, skills, roles or hooks to install, and its copy holds `ADAPTER.md`, `exports.yaml` and `README.md`. The generated deployment report identifies the actual paths and role names, and lists every copied file with its digest. diff --git a/docs/framework/agent-layers.md b/docs/framework/agent-layers.md index 644c7832..638238f3 100644 --- a/docs/framework/agent-layers.md +++ b/docs/framework/agent-layers.md @@ -49,6 +49,7 @@ How a model takes steps. Layer 2 holds only the framework's agents: | `codex` | The Codex CLI, working in a workspace | | `claude` | The Claude Code CLI, working in a workspace | | `pi` | The Pi CLI, working in a workspace (no roles, no web search) | +| `mini-swe-agent` | mini-swe-agent's bash loop, working in a workspace: one task through its Python API, in a process of its own (no roles, no web search) | API, the only one Layer 3 sees: @@ -69,7 +70,7 @@ agent, prompt = agent.instructed(system, prompt) # a copy that takes `system` environment, step by step in a resumed learning session. Codex and Claude Code resume their thread; direct and react carry the message history. - `roles`: named sub-agents (name, description, instructions) the agent may delegate to: Codex's roles, Claude - Code's agents. direct and react refuse them. + Code's agents. direct, react, pi and mini-swe-agent refuse them. - `trajectory`: the run as standard steps: messages, reasoning, commands with their exit codes and output, tool calls, file changes, errors. Layer 3 reads only these; a CLI's raw log (`agent-codex.jsonl`) is kept beside them as an archive. @@ -81,10 +82,12 @@ none), Codex's own sandbox mode, the CLI's home, the time limit (the full list i leaves the agent as it was. Running a task again when the provider could not serve it is Layer 3's: the workspace is placed anew first. -Codex, Claude Code and Pi share one runner, `CliAgent`: the isolation, the home, the sign-in, the tool server and the -process are the same for every CLI. Each CLI is a subclass whose hooks give what is its own: its command, its home, -and how its event stream is read (the reply, the trajectory, the usage, whether the provider served it, and what a -session that was not served met: `outcome`). A new CLI is +Codex, Claude Code, Pi and mini-swe-agent share one runner, `CliAgent`: the isolation, the home, the sign-in, the tool +server and the process are the same for every CLI. Each CLI is a subclass whose hooks give what is its own: its +command, its home, and how its event stream is read (the reply, the trajectory, the usage, whether the provider served +it, and what a session that was not served met: `outcome`). mini-swe-agent's process is the framework's own +(`hermit/agent/base/mini_swe_agent_run.py`): it runs one task through mini-swe-agent's Python API and writes the +session as JSON events, the stream the runner reads. A new CLI is a subclass listed in `CLIS`, a reader of its stream in `hermit/agent/base/trajectory.py`, and its model binding in Layer 1. @@ -142,6 +145,7 @@ Each Base Agent offers the tools to the model in its own way: | `direct` | Their schemas in the API request | Layer 3, which then calls `direct` again | | `react` | Their schemas in the API request | `react`, in its loop | | `codex`, `claude`, `pi` | The framework's tool server, over MCP (streamable HTTP) | The tool server | +| `mini-swe-agent` | One bash command, `hermit-tool ''`, listed in its system message | The tool server, through its JSON entrance | The tool server (`hermit/agent/base/tool_server.py`) is started for the one call and stopped after it. It carries a token in its URL, so that only the session it was started for can call it, and tells Codex to approve its tools @@ -156,7 +160,8 @@ inside the agent's sandbox with the agent's permissions. ## Limits `limits` states what the agent may touch. Each Base Agent enforces it with its own isolation, and refuses a limit it -cannot enforce (`Unsupported`) rather than ignore it. For Codex and Claude Code: +cannot enforce (`Unsupported`) rather than ignore it. For Codex, Claude Code, Pi and mini-swe-agent (Pi and +mini-swe-agent run in the run's sandbox or with no isolation, not in a container): | Limit | In a container | In the run's sandbox | No isolation | |---|---|---|---| diff --git a/docs/framework/commands.md b/docs/framework/commands.md index 025828d3..6e0a6601 100644 --- a/docs/framework/commands.md +++ b/docs/framework/commands.md @@ -71,12 +71,16 @@ user chooses: `--adapter none` runs the acting agent with no harness adapter at hermit deploy --adapter DIR --workspace WORKSPACE --host codex hermit deploy --adapter DIR --workspace WORKSPACE --host claude-code hermit deploy --adapter DIR --workspace WORKSPACE --host pi +hermit deploy --adapter DIR --workspace WORKSPACE --host mini-swe-agent ``` This generates host instruction files, role/skill files, and tool configuration. It does not launch the acting agent. Pi has no subagents and reads no hook file, so `--host pi` names an adapter's roles and hooks in `AGENTS.md` instead; its `.pi/` files (tools, skills) load once the user trusts the project in Pi, while `AGENTS.md` needs no -trust. `--context-limit BYTES` (Codex) is the `project_doc_max_bytes` your Codex configuration sets, 32768 (Codex's +trust. mini-swe-agent reads one config file, so `--host mini-swe-agent` writes `mini-swe-agent.yaml`: Hermit's +prompts over mini-swe-agent's `mini` config, with the context, the names of the adapter's skills, roles, hooks and +other files, and its tools as one bash command (`hermit-tool`) in the system template; run `mini -c +mini-swe-agent.yaml` from the workspace ([tutorial](../tutorials/mini-swe-agent.md)). `--context-limit BYTES` (Codex) is the `project_doc_max_bytes` your Codex configuration sets, 32768 (Codex's default) when not given: a longer `AGENTS.md` is refused, since Codex would cut it; `0` skips the check. It is the per-project way to use an adapter; the per-agent way, `convert`, and how the two compare are in [What is a harness adapter?](what-is-a-harness-adapter.md#two-ways-to-use-an-adapter). The workspace must be an empty directory or one previously generated for the same host, separate from the adapter directory. See the [file mapping](adapter.md#acting-interface) for exact paths and update behavior. diff --git a/docs/framework/configuration.md b/docs/framework/configuration.md index 7ac70412..71a65597 100644 --- a/docs/framework/configuration.md +++ b/docs/framework/configuration.md @@ -13,7 +13,7 @@ described in the recipe; this file alone does not supply them. :caption: examples/officeqa/gepa/smoke.yaml ``` -`agent.name` names the acting agent: a built-in one (`react`, `codex`, `claude-code`) or the task's own class or +`agent.name` names the acting agent: a built-in one (`react`, `codex`, `claude-code`, `pi`, `mini-swe-agent`) or the task's own class or function. `judge.path` names its scoring module. `template` chooses the adapter's files and editing rules. `evolver.name` chooses the learning method. The `llm` section configures the run's model tool, which the GEPA learner uses. @@ -50,7 +50,7 @@ Use the same run name for training and evaluation. Unknown configuration keys ar |---|---| | `data` | Task JSONL file, split files, tasks per step, whether to shuffle the train split (`shuffle`, default true), and shuffle seed. | | `task_info` | Directory explaining the task and objective to the learner. | -| `agent.name` | A built-in agent (`react`, `codex`, `claude-code`), `file.py:Class` for the task's own `Agent` subclass, or `file.py:function` for a plain `run(task, adapter)`. `pipeline.run` is the older spelling. | +| `agent.name` | A built-in agent (`react`, `codex`, `claude-code`, `pi`, `mini-swe-agent`), `file.py:Class` for the task's own `Agent` subclass, or `file.py:function` for a plain `run(task, adapter)`. `pipeline.run` is the older spelling. | | `agent.concurrency` | Maximum concurrent task executions during rollout and evaluation. | | `agent.params` | The agent's settings: for a built-in agent its model, prompt and tools; for the task's own agent whatever it reads. | | `judge.path`, `judge.name` | Scoring module/file and function name; the default name is `score`. | diff --git a/docs/framework/task.md b/docs/framework/task.md index 0c4a1852..442aeda0 100644 --- a/docs/framework/task.md +++ b/docs/framework/task.md @@ -87,10 +87,11 @@ agent: |---|---|---| | `react` | a chat-completions tool-calling loop over the task's tools and the adapter's tools | `model`, `prompt`, `tools`, `user`, `max_turns`, `max_output_tokens`, `max_tokens`, `temperature`, `answer_tag`, `tool_result_chars`, `logs` | | `codex`, `claude-code`, `pi` | Codex, Claude Code or Pi run in a fresh workspace the adapter is deployed into (Pi: no `login`, no roles; its project's `.pi/` is trusted for the session) | `model`, `login`, `effort`, `prompt`, `files`, `mounts`, `prepare`, `answer`, `collect`, `context_query`, `project_doc_max_bytes`, `args`, `env`, `price`, `timeout_s`, `retries`, `retry_wait_s`, `workspaces` | +| `mini-swe-agent` | mini-swe-agent run in a fresh workspace the adapter is deployed into (no `login`, no roles; [tutorial](../tutorials/mini-swe-agent.md)) | `model`, `model_class`, `config`, `step_limit`, `effort`, `prompt`, `files`, `mounts`, `prepare`, `answer`, `collect`, `context_query`, `env`, `price`, `timeout_s`, `retries`, `retry_wait_s`, `workspaces` | ### A coding-agent CLI as the acting agent -`codex` and `claude-code` are the framework's one way to act with a coding-agent CLI: a task gives what is its own, +`codex`, `claude-code`, `pi` and `mini-swe-agent` are the framework's one way to act with a coding-agent CLI: a task gives what is its own, in configuration and, where its workspace is built in code, in functions it names (`file.py:function`); the agent does the rest. OfficeQA's Codex runs are one such configuration (`examples/officeqa/utils/task.py`): diff --git a/docs/framework/what-is-a-harness-adapter.md b/docs/framework/what-is-a-harness-adapter.md index c29512b5..e8363daa 100644 --- a/docs/framework/what-is-a-harness-adapter.md +++ b/docs/framework/what-is-a-harness-adapter.md @@ -32,7 +32,7 @@ manifest-driven service lifecycle remains a [proposal](../developer/service-desi | Route | Output | Use when | |---|---|---| -| `deploy` | A Codex, Claude Code, or Pi workspace | You want project instructions, skills, roles, and tool configuration | +| `deploy` | A Codex, Claude Code, Pi, or mini-swe-agent workspace | You want project instructions, skills, roles, and tool configuration | | `convert` | A native plugin package for a supported target | You want host-native packaging and can accept that target's export limits | | Custom pipeline | Your agent receives the exports directly | You own the acting loop or need dynamic integration | diff --git a/docs/get-started/installation.md b/docs/get-started/installation.md index a8c88366..ab9d2655 100644 --- a/docs/get-started/installation.md +++ b/docs/get-started/installation.md @@ -87,6 +87,16 @@ PyPI's torch for Linux is the CUDA build; for the CPU one, install torch first f is shown to the sandboxes too (`sandbox.hide` covers a benchmark's scoring module; an editable install's checkout is never shown). +## Install mini-swe-agent + +The `mini-swe-agent` acting agent and `hermit deploy --host mini-swe-agent` need mini-swe-agent (2.4.6, which brings +litellm), published as an extra: + +```bash +python -m pip install -e ".[mini-swe-agent]" # a checkout +python -m pip install "hermit-harness[mini-swe-agent]" +``` + ## Choose the additional requirements | Workflow | Additional requirements | @@ -94,6 +104,7 @@ never shown). | [Offline first training](quickstart.md) | None beyond the core install | | [First real-model training](first-model.md) | `python -m pip install -e . --group gepa`, a compatible chat-completions endpoint, model id, and credential; data is included | | Generate a Codex / Claude Code / Pi workspace | No host CLI is needed until you start the host | +| Act with, or deploy to, mini-swe-agent | The `mini-swe-agent` extra and a chat-completions model ([tutorial](../tutorials/mini-swe-agent.md)) | | Run an acting host | Installed and authenticated host, plus adapter tool dependencies | | Prompt-only learner | The Codex CLI, Linux with bubblewrap, and a learner backend that serves the Responses API | | GEPA | `python -m pip install -e . --group gepa`, learner endpoint, validation split | diff --git a/docs/guides/model-backends.md b/docs/guides/model-backends.md index 44bc10f5..dcbd316f 100644 --- a/docs/guides/model-backends.md +++ b/docs/guides/model-backends.md @@ -25,6 +25,7 @@ hermit train --config run.yaml model=vllm/Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 | `llm` (or `evolver.model` with `evolver.base: direct`) | the Evolver's `direct` agent and the run's `llm` tool: the `gepa`, `skillopt`, `ace`, `reharness` and `continual_harness` Evolvers | chat completions | | `agent.params.model` (or `agent.model`) with `agent.name: react` | the built-in tool-calling agent | chat completions | | `agent.params.model` (or `agent.model`) with `agent.name: codex` / `claude-code` | the CLI acting agent | Responses API / Messages API | +| `agent.params.model` (or `agent.model`) with `agent.name: mini-swe-agent` | mini-swe-agent, through litellm | chat completions | | `evolver.params.model` of the `codex`, `claude` and `mwm` Evolvers | the learner's CLI session | Responses API / Messages API | | `agent.params.model` with `agent.name: pi`, `evolver.params.model` of the `pi` Evolver | Pi | chat completions, else the Messages API, else the Responses API: the first the backend serves | | `model` (top level) | the default for all of the above when they name none | | diff --git a/docs/index.md b/docs/index.md index a7679204..9612e1f8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -60,6 +60,7 @@ guides/troubleshooting tutorials/your-own-task tutorials/customize-an-adapter tutorials/deploy-an-adapter +tutorials/mini-swe-agent ``` ```{toctree} diff --git a/docs/reference/compatibility.md b/docs/reference/compatibility.md index 9277d6d4..69b6f42b 100644 --- a/docs/reference/compatibility.md +++ b/docs/reference/compatibility.md @@ -12,7 +12,7 @@ numbers alone may not distinguish unreleased changes. | Context-only workspace generation | macOS and Linux | | Sandboxed learner programs | Linux with bubblewrap and permitted namespaces | | Built-in methods | Prompt-only (the Codex learner on the prompt-only template), Codex, Claude, Pi, GEPA, SkillOpt, MWM, Harness², Continual Harness, ReHarness, Meta-Harness Adapter, ACE, AHE Adapter, RRSI Adapter, EvoSafeHarness, OPRO | -| Workspace deployment | Codex, Claude Code, and Pi | +| Workspace deployment | Codex, Claude Code, Pi, and mini-swe-agent | | A subscription instead of an API key | Codex (`login: chatgpt`) and Claude Code (`login: claude`, CLI 2.1.248 or later, tested on 2.1.318), acting and learning; see [Claude subscription](../guides/claude-subscription.md) | | Native plugin conversion | Claude Code, Codex, Pi, Hermes, with export restrictions | | Stdio tool server | `hermit serve DIR`; normally started by the host | @@ -39,7 +39,7 @@ framework's stdio server. Neither command guarantees that arbitrary tool depende - Budget observation can overshoot with in-flight work; missing usage remains unknown. - Crash recovery restores the last completed framework step, not arbitrary external service state. - The evaluator's `solved_once` threshold is fixed at score 1.0. -- Per-task adapter context comes only through `exports.yaml`'s `context_tool`, consumed by `Session`, the `react` agent and the `codex` and `claude-code` agents; its template must declare `plugin-portable: false`, and `deploy` and `convert` refuse such adapters. +- Per-task adapter context comes only through `exports.yaml`'s `context_tool`, consumed by `Session`, the `react` agent and the `codex`, `claude-code`, `pi` and `mini-swe-agent` agents; its template must declare `plugin-portable: false`, and `deploy` and `convert` refuse such adapters. - The proposed manifest `serve` section, health checks, standalone startup, and relay lifecycle are not supported. - A model reached through a CLI (`llm: {cli: ...}`, `agent.model: {cli: ...}`) takes text and gives text: tool calls, sampling and structured-output requests are refused ([the model contract](../guides/claude-subscription.md#the-model-contract)). A subscription has no per-token cost, so `budget.usd` bounds nothing on it. - Harness²'s editors are told of upstream's Codex surface (the `harness2-codex` template); a Claude Code acting agent runs the same tree in its own places, and its execution's stream reaches them rendered in upstream's grammar by `harness2_upstream/render.py`. diff --git a/docs/reference/config-fields.md b/docs/reference/config-fields.md index 1b26aa1b..19f84ced 100644 --- a/docs/reference/config-fields.md +++ b/docs/reference/config-fields.md @@ -32,7 +32,7 @@ guarantee that their consumer supports arbitrary fields. | `data.shuffle` | boolean / true | Shuffle training order; set false for ordered streams | | `data.shuffle_seed` | integer / 0 | Training shuffle seed; does not seed every model or method | | `task_info` | directory / null | Learner-facing task description; provide `README.md` | -| `agent.name` | string / null; required | Built-in `react`, `codex`, `claude-code`, `pi`, `harbor` (a benchmark's task set in Harbor's format, see [Benchmarks](../framework/task.md#benchmarks)), an Agent class, or a `file.py:function` / `module:function` | +| `agent.name` | string / null; required | Built-in `react`, `codex`, `claude-code`, `pi`, `mini-swe-agent`, `harbor` (a benchmark's task set in Harbor's format, see [Benchmarks](../framework/task.md#benchmarks)), an Agent class, or a `file.py:function` / `module:function` | | `agent.concurrency` | integer / 4 | Maximum concurrent rollout/evaluation task executions; use a positive value | | `agent.timeout_min` | number / null | A sandboxed task call (its worker) running longer is stopped and recorded as an error; null: no limit | | `agent.params` | mapping / {} | Passed to Agent constructors; plain functions receive it when they declare a `params` argument | @@ -79,8 +79,10 @@ dialects it serves, prices). See [Model backends](../guides/model-backends.md). | `llm` | reference, mapping / null | The run's chat-completions `llm` tool: a reference, `{model: , price, extra_body}`, the endpoint itself (below), or a CLI's sessions, `{cli: ..., ...}` (below) | `agent.params.model` takes the same forms. For a Codex or Claude Code acting agent or Evolver a reference becomes -the CLI's provider settings (`args` for Codex, `env` for Claude Code, `provider` for Pi) and the model's id; a plain id without a -backend goes to the CLI's own provider as before. +the CLI's provider settings (`args` for Codex, `env` for Claude Code, `provider` for Pi) and the model's id; a +mini-swe-agent acting agent calls the backend through litellm with the key passed per request. A plain id without a +backend goes to the CLI's own provider as before (for mini-swe-agent: a litellm model id, its key from litellm's +usual variable). ### Learner model tool diff --git a/docs/tutorials/mini-swe-agent.md b/docs/tutorials/mini-swe-agent.md new file mode 100644 index 00000000..c0877684 --- /dev/null +++ b/docs/tutorials/mini-swe-agent.md @@ -0,0 +1,126 @@ +# Use mini-swe-agent as the acting agent + +[mini-swe-agent](https://mini-swe-agent.com) is a minimal agent: its model runs bash commands and nothing else, and +one YAML config's templates make its messages. In Hermit it is a Base Agent beside Codex and Claude Code +([Agent layers](../framework/agent-layers.md)): `agent.name: mini-swe-agent` acts in `check`, `eval` and `train`, and +`hermit deploy --host mini-swe-agent` writes a workspace you run `mini` in. With Hermit's prompts both give the +agent the same system message, so a deployed adapter reads what it read in training. + +## 1. Install + +From the checkout root, in the environment that runs Hermit: + +```bash +python -m pip install -e ".[mini-swe-agent]" +``` + +The extra pins mini-swe-agent 2.4.6, which brings litellm. + +## 2. How an adapter reaches the agent + +The acting agent deploys the adapter into each task's workspace as `hermit deploy --host mini-swe-agent` does: the +whole adapter is copied to `hermit//`, and `mini-swe-agent.yaml` holds what mini-swe-agent can take. + +| Adapter part | What mini-swe-agent gets | +|---|---| +| Context (`ADAPTER.md`), and a `context_tool`'s text for the task | The system message, under "Instructions from the harness adapter" | +| Tools | One bash command, `hermit-tool ''`; the system message lists each tool with its argument schema | +| Skills, roles, hooks, other files | Named in the system message with their paths in `hermit//` and what to do with them: mini-swe-agent loads no skills, starts no sub-agents and runs no hooks | + +`hermit-tool` is not a program on `PATH`: Hermit's environment class recognizes a command whose first word is +`hermit-tool` and answers it in mini-swe-agent's process. It must be a command of its own; `hermit-tool` alone lists +the tools. Every other command runs as in mini-swe-agent's local environment. + +## 3. Configure the acting agent + +```yaml +model: deepseek/deepseek-flash # /, see Model backends +agent: + name: mini-swe-agent + concurrency: 3 + params: + web_search: false # set either way for every CLI agent; mini-swe-agent has none + model_class: litellm # or litellm_textbased: one ```mswea_bash_command block per response + # (a / reference takes these two only) + config: agent/mini.yaml # optional: a mini-swe-agent config merged over Hermit's + step_limit: 50 # model calls per task (the config's agent.step_limit wins) + prompt: "{question}" # the task text (default: the task's prompt, question, instruction or input) + files: [environment/data] # copied into every workspace before the run + env: {} # variables for the agent's commands (a `$NAME` value is expanded) + timeout_s: 3600 +``` + +The per-layer form says the same: `agent.base: mini-swe-agent`, the model as `agent.model`, and mini-swe-agent's own +settings in `agent.mini-swe-agent`. The task functions of the [CLI agents](../framework/task.md) work here too: +`prepare`, `answer`, `collect`, `mounts`, `context_query`. + +Each task runs in a fresh workspace (`work/`, with the log in `log/` beside it). The agent submits its answer as +mini-swe-agent ends a task, `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT && cat answer.txt`, and the submission is the +last message the judge scores; without one (the step limit, the time limit) it is the last assistant message. The +log holds the session's events (`agent-mini-swe-agent.jsonl`), which every Evolver reads as steps: the messages, each +command with its output and exit code, errors. + +The model is a [backend reference](../guides/model-backends.md); mini-swe-agent calls it through litellm, at the +backend's URL with its key. The key is passed with each request and written nowhere; the agent's commands see `env` +and a few basic variables (`PATH`, `HOME`, `LANG`, ...) only, and on Linux mini-swe-agent's process is made +non-dumpable, so a command cannot read its environment either (on macOS a command run as you can, with `ps eww`). +That keeps the key out of mini-swe-agent's commands and process, not out of every process that holds it: with the +sandbox off, Hermit's own process (started with the key exported) runs as you too, as for every CLI agent. Only the +run's sandbox confines the commands (`sandbox.enabled`, Linux with bubblewrap): there the agent reaches its model and +nothing else. + +Tokens are read from the responses and priced at the backend's `prices`; without a price the cost is unknown. +mini-swe-agent's own cost tracking is off, so its `cost_limit` never stops a task: `step_limit` and `timeout_s` do +(at `timeout_s` the process is stopped), and one request is bounded by 600 seconds. A failed model call (a context +overflow) fails the session, as a CLI's failed session does: no answer is judged, the task is recorded as an error, and +its tokens still count; a provider that could not serve (a rate limit, an outage, past mini-swe-agent's own retries) +runs the task again (`retries`). A setting that would fail every task (no model, a `config` that names the model, a +wrong key or model id found by the first call) starts no session and is reported as the agent's error, not as an +answer. At `timeout_s` the command that is running is stopped too. + +`config` is a mini-swe-agent config file merged over Hermit's the way `mini -c` merges files: change +`agent.step_limit`, the templates, `model.model_kwargs` (temperature, `max_tokens`) or the `environment` section +(`timeout` of a command, more `env`). A system template of your own receives the adapter's block where it says +`{{ adapter_context }}`, or at its end. The model is the Base Agent's and the environment Hermit's local one: the +file may not set `model.model_name`, `model.model_class` or `environment.environment_class`. With the sandbox on, +declare the file in `environment.read`, as the task's own files are: the task sandbox shows nothing else of the run's +directory. + +## 4. Run the example + +`docs/_examples/mini_swe_agent/` has nine questions about a small CSV file and a judge that accepts exactly +`{"answer": }`, with GEPA as the learner (`--group gepa`). It uses DeepSeek for both: + +```bash +python -m pip install -e ".[mini-swe-agent]" --group gepa +export DEEPSEEK_API_KEY=... +mkdir -p .demo && cp -R docs/_examples/mini_swe_agent .demo/mini-swe-agent +hermit check --config .demo/mini-swe-agent/run.yaml +hermit eval --config .demo/mini-swe-agent/run.yaml --adapter none +hermit train --config .demo/mini-swe-agent/run.yaml +hermit eval --config .demo/mini-swe-agent/run.yaml --adapter final +``` + +Another backend is one override, for example `model=openai/`. The tasks need the file and a few commands; +the format is what there is to learn, and a strong model may answer in it from the start. **There is no promised +score gain.** The example sets `sandbox.enforce: best_effort`: with bubblewrap each task runs in the task sandbox; +without it (macOS) the run is unconfined, recorded and warned. + +## 5. Deploy to a mini-swe-agent workspace + +```bash +hermit deploy --adapter .demo/mini-swe-agent/runs/mini-swe-agent/checkout --workspace .demo/mini-workspace \ + --host mini-swe-agent +cd .demo/mini-workspace +mini -c mini-swe-agent.yaml -m deepseek/deepseek-flash +``` + +The workspace holds `mini-swe-agent.yaml` (Hermit's prompts over mini-swe-agent's `mini` config, the adapter's block +written into the system template), the adapter's copy under `hermit//` and `.hermit-deploy.json`. An adapter with +tools also names Hermit's environment class and the adapter in the config's `environment` section, and every tool +call is appended to `tool-calls.jsonl`; run `mini` from the Python environment Hermit is installed in. The model and +the limits are mini-swe-agent's own settings (`-m`, its keys, global config and `cost_limit`), as for any `mini` run. +The deployment uses Hermit's prompts for the `mini` config; the acting agent's `model_class` and `config` are not +part of it: give `mini` those yourself (`-c mini-swe-agent.yaml -c agent/mini.yaml`), keeping its system template, +which holds the adapter's block. Redeploying replaces the config; the workspace must be empty or generated for the +same host. diff --git a/docs/web/index.html b/docs/web/index.html index f7af611f..705c4832 100644 --- a/docs/web/index.html +++ b/docs/web/index.html @@ -145,7 +145,7 @@

§What Hermit gives you out of the box

  • One Trainer for every method: rollouts, judge feedback, an Evolver step, a checked commit, the next version
  • Built-in evolution methods and a shared interface for your own Evolver
  • Adapters as files in a git tree: inspect every version, roll back, ship the final one
    • -
  • Deploy workspaces for Codex, Claude Code or Pi, or export supported contents as a native plugin for compatible hosts
    • +
  • Deploy workspaces for Codex, Claude Code, Pi or mini-swe-agent, or export supported contents as a native plugin for compatible hosts
  • Adapter specification checks and versioned updates, with sandboxing for supported learner programs
  • Any model backend by reference: Fireworks, OpenAI, Anthropic, DeepSeek, vLLM, SGLang, Ollama
    • diff --git a/hermit/agent/acting/__init__.py b/hermit/agent/acting/__init__.py index f696101f..0e076d64 100644 --- a/hermit/agent/acting/__init__.py +++ b/hermit/agent/acting/__init__.py @@ -3,7 +3,7 @@ | `agent.name` | The agent | |---|---| | `react` | a chat-completions tool-calling loop over the task's @tool functions and the adapter's tools | -| `codex`, `claude-code` | the agent CLI in a workspace the adapter is deployed into | +| `codex`, `claude-code`, `pi`, `mini-swe-agent` | the agent CLI in a workspace the adapter is deployed into | | `agent/agent.py:MyAgent` | the task's own `Agent` subclass (a benchmark's loop with the adapter plugged in, ...) | | `agent/pipeline.py:run` | a plain `run(task, adapter)` function, wrapped | diff --git a/hermit/agent/acting/agent.py b/hermit/agent/acting/agent.py index 8cafd23b..6d827c63 100644 --- a/hermit/agent/acting/agent.py +++ b/hermit/agent/acting/agent.py @@ -7,7 +7,7 @@ def run(self, task: dict, adapter: Path | None) -> RunOutput A config names the agent: agent: - name: react # a built-in: react | codex | claude-code | pi + name: react # a built-in: react | codex | claude-code | pi | mini-swe-agent name: agent/agent.py:CLBenchAgent # or the task's own class (file.py:Class, module:Class) name: agent/pipeline.py:run # or a plain function run(task, adapter), wrapped as a FunctionAgent concurrency: 4 @@ -75,9 +75,10 @@ def builtin(name: str): if name == "react": from hermit.agent.acting.react import ReactAgent return ReactAgent - if name in ("cli", "codex", "claude-code", "claude", "pi"): + if name in ("cli", "codex", "claude-code", "claude", "pi", "mini-swe-agent"): from hermit.agent.acting.workspace import CliAgent - backend = {"codex": "codex", "claude-code": "claude", "claude": "claude", "pi": "pi"}.get(name) + backend = {"codex": "codex", "claude-code": "claude", "claude": "claude", "pi": "pi", + "mini-swe-agent": "mini-swe-agent"}.get(name) if backend is None: return CliAgent return lambda params, base: CliAgent({"backend": backend, **(params or {})}, base) @@ -87,7 +88,7 @@ def builtin(name: str): return None -BUILTIN_NAMES = ("react", "codex", "claude-code", "pi", "cli", "harbor") +BUILTIN_NAMES = ("react", "codex", "claude-code", "pi", "mini-swe-agent", "cli", "harbor") def load_agent(name: str, params: dict | None, base: Path) -> Agent: diff --git a/hermit/agent/acting/workspace.py b/hermit/agent/acting/workspace.py index f280c344..af6c125b 100644 --- a/hermit/agent/acting/workspace.py +++ b/hermit/agent/acting/workspace.py @@ -1,8 +1,9 @@ -"""The `codex`, `claude-code` and `pi` agents: one task run by the agent CLI in a workspace the adapter is deployed -into. They are the framework's one way to act with a coding-agent CLI; a task adds only what is its own. +"""The `codex`, `claude-code`, `pi` and `mini-swe-agent` agents: one task run by the agent CLI in a workspace the +adapter is deployed into. They are the framework's one way to act with a coding-agent CLI; a task adds only what is +its own. agent: - name: codex # or claude-code; `cli` with params.backend is the same + name: codex # or claude-code, pi or mini-swe-agent (`cli` with params.backend) params: model: gpt-5.4 # optional: the CLI's default otherwise. A `/` reference # (hermit.agent.model.backends) binds the CLI to that backend: its provider @@ -38,6 +39,8 @@ retries: 3 # runs the provider could not serve are run again, then an error retry_wait_s: 120 workspaces: .logs/workspaces # one fresh directory per task, beside the config (`~` is expanded) + # mini-swe-agent only (hermit.agent.base.cli.MiniSweAgent): model_class (litellm or litellm_textbased), + # config (its config file, beside the run config), step_limit (model calls per task, default 50) For every task a new directory holds the workspace (`work/`), the CLI's home (`home/`, Codex) and the log (`log/`). The adapter is deployed into the workspace as `hermit deploy` deploys it (hermit.hosts): the whole adapter under @@ -69,7 +72,7 @@ from hermit.agent.base.login import LOGINS from hermit.task import RunOutput, load_callable -BACKENDS = ("codex", "claude", "pi") +BACKENDS = ("codex", "claude", "pi", "mini-swe-agent") DEPLOYS = ("host",) ISOLATIONS = ("none", "container") INPUT_FIELDS = ("question", "prompt", "instruction", "input", "query") @@ -144,6 +147,9 @@ def deploy(adapter: Path, workspace: Path, backend: str, *, context: str | None if backend == "pi": from hermit.hosts.pi import prepare as prepare_pi return prepare_pi(adapter, workspace, sys.executable, context=context) + if backend == "mini-swe-agent": + from hermit.hosts.mini_swe_agent import prepare as prepare_mini + return prepare_mini(adapter, workspace, sys.executable, context=context, serve_tools=serve_tools) from hermit.hosts.claude_code import prepare return prepare(adapter, workspace, sys.executable, context=context, serve_tools=serve_tools) @@ -156,9 +162,10 @@ def _deploy(task: dict, adapter: Path | None, work: Path, backend: str, params: from hermit.adapter.loader import agent_tools, load_manifest from hermit.adapter.manifest import PORTABLE manifest = load_manifest(adapter) - # A container session cannot start the tools' server on this machine: the session's own tool server (the Base - # Agent's, which a container reaches) serves them instead. - harness_tools = params.get("isolation") == "container" and bool(agent_tools(manifest)) + # A container session cannot start the tools' server on this machine, and mini-swe-agent reaches tools only + # through its Base Agent: the session's own tool server (the Base Agent's) serves them instead. + harness_tools = (params.get("isolation") == "container" or backend == "mini-swe-agent") and \ + bool(agent_tools(manifest)) # An adapter only hermit runs (not plugin-portable: a context tool's per-task context, say) has no installed # workspace, so `hermit deploy` refuses it; here the task is known, so its context is deployed. context, record = _context(task, adapter, params) if not manifest.get(PORTABLE, True) else (None, None) @@ -176,6 +183,8 @@ def _session_args(backend: str, deployed: dict, work: Path) -> list[str]: if backend == "codex": return ["-c", f'projects={{"{work}" = {{ trust_level = "trusted" }} }}', *(["--enable", "multi_agent"] if deployed.get("roles") else [])] + if backend == "mini-swe-agent": # it reads the deployment's config from the workspace itself + return [] return ["--mcp-config", str(work / ".mcp.json")] if deployed.get("server_name") else [] @@ -277,11 +286,11 @@ def _check(backend: str, params: dict) -> str: if not isinstance(params.get("web_search"), bool): raise ValueError("agent.params.web_search must be set, true or false: Codex's hosted search runs at the " "provider, past any sandbox or proxy, so it is never left to a default") - if backend == "pi": # Pi signs in through a model reference, and runs on this machine + if backend in ("pi", "mini-swe-agent"): # they sign in through a model reference, and run on this machine if params.get("login") is not None: - raise ValueError("agent pi takes no login: its model is a / reference") + raise ValueError(f"agent {backend} takes no login: its model is a / reference") if params.get("isolation") == "container": - raise ValueError("isolation: container runs Codex or Claude Code; agent pi takes isolation: none") + raise ValueError(f"isolation: container runs Codex or Claude Code; agent {backend} takes isolation: none") return backend login, label = params.get("login"), {"codex": "Codex", "claude": "Claude Code"}[backend] if login is not None and login not in LOGINS: @@ -307,7 +316,7 @@ def base_agent(backend: str, params: dict) -> BaseAgent: settings = {k: params[k] for k in ("model", "binding", "login", "plan_dir", "plan_max_wait_s", "usage_credits", "effort", "isolation", "container", "codex_sandbox", "shell_env", "project_doc_max_bytes", "args", "env", "price", "provider", - "image_input") + "image_input", "model_class", "config", "step_limit") if k in params and (params[k] is not None or k == "plan_max_wait_s")} if params.get("claude_permission_mode"): settings["permission_mode"] = params["claude_permission_mode"] @@ -316,6 +325,8 @@ def base_agent(backend: str, params: dict) -> BaseAgent: def _run(task: dict, adapter: Path | None, params: dict, base: Path) -> RunOutput: backend = _check(params.get("backend", "codex"), params) + if params.get("config"): # mini-swe-agent's config file, beside the run config + params = {**params, "config": str(Path(base) / os.path.expanduser(str(params["config"])))} agent = base_agent(backend, params) root = Path(os.path.expanduser(str(params.get("workspaces", ".logs/workspaces")))) # Resolved: a container binds each directory at its real path, and the CLI is given these paths. diff --git a/hermit/agent/base/cli.py b/hermit/agent/base/cli.py index ef6e2152..1c1c5a3e 100644 --- a/hermit/agent/base/cli.py +++ b/hermit/agent/base/cli.py @@ -1,4 +1,5 @@ -"""`codex`, `claude` and `pi`: a coding-agent CLI as a Base Agent (docs/framework/agent-layers.md#layer-2-base-agent). +"""`codex`, `claude`, `pi` and `mini-swe-agent`: a coding-agent CLI as a Base Agent +(docs/framework/agent-layers.md#layer-2-base-agent). One `run` is one session of the CLI in print mode: the prompt is given, the CLI works in `workspace` with its own tools and the call's `tools` (served over MCP by the framework's tool server), and its last message is the reply. @@ -69,6 +70,8 @@ image_input (true: the bound model takes images, such as a tool's result; default: text only, Pi's default for a model it does not know), append_system_prompt, effort (its --thinking level). Pi takes no roles, has no web search and runs under isolation none or sandbox. + mini-swe-agent only: model_class, config, step_limit (MiniSweAgent; hermit.agent.base.mini_swe_agent). It takes no + roles, no session and no login, has no web search and runs under isolation none or sandbox. Limits are enforced by the isolation, or the call is refused (Unsupported): @@ -96,9 +99,9 @@ from hermit.agent.base import container, login as codex_login from hermit.agent.base.interface import BaseAgent, Limits, Mount, Result, Unsupported -from hermit.agent.base.records import parse_claude, parse_codex, parse_pi, price_usage +from hermit.agent.base.records import parse_claude, parse_codex, parse_mini_swe_agent, parse_pi, price_usage from hermit.agent.base.tool import Tool -from hermit.agent.base.trajectory import claude_steps, codex_steps, events, pi_steps +from hermit.agent.base.trajectory import claude_steps, codex_steps, events, mini_swe_agent_steps, pi_steps from hermit.agent.model.backends import CLAUDE_ENV ISOLATIONS = ("none", "sandbox", "container") @@ -1462,8 +1465,160 @@ def _pi_session(home: Path, handle) -> Path | None: return sorted(found)[-1] if found else None +class MiniSweAgent(CliAgent): + """mini-swe-agent (hermit.agent.base.mini_swe_agent): one task in a process of its own + (hermit.agent.base.mini_swe_agent_run), whose stdout is the stream. + + Settings (given once; `configured(...)` for a call's own), besides CliAgent's isolation, home, timeout_s, env, log, + bin and stdin: + model a litellm model id (the provider's key from its usual variable, or from mini-swe-agent's own + global config, whose `.env` is copied into the session's home), unless `binding` is given + binding the resolved `/` reference (hermit.agent.model.backends): its URL, key variable, + extra body and price + model_class litellm (its bash tool, the default) or litellm_textbased (one ```mswea_bash_command block per + response); without a binding, any mini-swe-agent model class name + effort sent as litellm's reasoning_effort + config a mini-swe-agent config file merged over Hermit's (agent.step_limit, the templates, + model.model_kwargs, environment.timeout, ...); not its model name or class, nor its environment + class + step_limit model calls per task (default 50; the config's agent.step_limit wins) + price USD per million tokens (absent: the binding's; without one the cost is unknown) + web_search must be false: mini-swe-agent has no web search + + mini-swe-agent has no sub-agents and no resumable session: a call with `roles` or `session` is refused + (Unsupported), as one under a ChatGPT login is. It runs without isolation or in the run's sandbox, not in a + container. + """ + name = backend = "mini-swe-agent" + label = "mini-swe-agent" + program = sys.executable # the runner is a module of this framework, run by its interpreter + isolations = ("none", "sandbox") + home_var, user_folder = "MSWEA_GLOBAL_CONFIG_DIR", ".mini-swe-agent" + home_files = (".env",) # its global config: the keys `mini-extra config` keeps + binding_classes = ("litellm", "litellm_textbased") # the model classes that call a binding's endpoint (chat) + turn_done = "message" # an assistant's, which `served` looks for + + def __init__(self, settings: dict | None = None, model: Any = None): + super().__init__({"stdin": True, "tools_env": "HERMIT_TOOLS_URL", **(settings or {})}, model) + + @classmethod + def user_home(cls) -> Path: + """mini-swe-agent's own global config directory: MSWEA_GLOBAL_CONFIG_DIR, else platformdirs' user config + dir.""" + if os.environ.get("MSWEA_GLOBAL_CONFIG_DIR"): + return Path(os.environ["MSWEA_GLOBAL_CONFIG_DIR"]) + try: + from platformdirs import user_config_dir + except ImportError: # mini-swe-agent, which brings platformdirs, is not installed + return Path.home() / ".config" / "mini-swe-agent" + return Path(user_config_dir("mini-swe-agent")) + + def run(self, prompt, *, workspace=None, tools: Sequence[Tool] = (), limits: Limits | None = None, + session: Any = None, roles: Sequence[dict] = ()) -> Result: + if session is not None: + raise Unsupported("mini-swe-agent cannot resume a session") + if roles: + raise Unsupported("mini-swe-agent has no sub-agents: it takes no roles") + if self.settings.get("login"): + raise Unsupported("mini-swe-agent has no login: give a / reference or a litellm model") + if self.settings.get("web_search"): + raise Unsupported("mini-swe-agent has no web search: set web_search false") + if not (self.settings.get("binding") or self.settings.get("model")): + raise ValueError("mini-swe-agent needs a model: a / reference or a litellm model id") + model_class = self.settings.get("model_class") or "litellm" + if self.settings.get("binding") and model_class not in self.binding_classes: + # another class sends what it is given to its own service: the binding's key and URL among it + raise Unsupported(f"mini-swe-agent's model_class {model_class!r} calls its own service; a " + f"/ reference takes {' or '.join(self.binding_classes)}") + return super().run(prompt, workspace=workspace, tools=tools, limits=limits, session=session, roles=roles) + + def prepare_home(self, home, roles, tools_url, tools_name): + """The task's settings for the runner, in its home: no key (the runner reads the key's variable), and of + `env` only the names (the runner passes those variables to the commands).""" + s = self.settings + run = {k: s[k] for k in ("model", "binding", "model_class", "effort", "step_limit", "system") + if s.get(k) is not None} + if s.get("config"): + run["config"] = str(Path(os.path.expanduser(str(s["config"]))).resolve()) + run["env_names"] = sorted((s.get("env") or {})) + path = home / "mini-swe-agent.json" + if os.path.lexists(path) and not _regular(path): + os.unlink(path) + path.write_text(json.dumps(run, ensure_ascii=False)) + + def command(self, prompt, workspace, last_file, *, session=None, roles=(), tools_url=None): + s = self.settings + # -P: the workspace, its working directory, is not on its import path (a task's yaml.py is no `yaml`) + return [str(s.get("bin") or self.program), "-P", "-m", "hermit.agent.base.mini_swe_agent_run", + "--settings", str(last_file.with_name("mini-swe-agent.json")), "--cwd", str(s.get("cwd") or workspace), + "--last", str(last_file), *(["--stdin"] if s.get("stdin") else ["--prompt", prompt]), + *(s.get("args") or [])] + + @classmethod + def web_search_args(cls, enabled: bool) -> list[str]: + return [] + + def system_settings(self, system, prompt): + """System instructions go at the end of the system message, with the adapter's block.""" + return {"system": system}, prompt + + @classmethod + def served(cls, raw: str) -> bool: + """Whether the provider served the session: it answered, and did not stop serving before the end (an error + the process marks `unavailable`, as Codex's turn that fails is no turn completed).""" + objects = _objects(raw)[0] + return any(e.get("type") == cls.turn_done and (e.get("message") or {}).get("role") == "assistant" + for e in objects) and not any(e.get("type") == "error" and e.get("unavailable") for e in objects) + + @classmethod + def errors(cls, event): + """The process's error; one from a provider that could not serve reads as such to UNAVAILABLE.""" + if event.get("type") != "error": + return [] + return [("service unavailable: " if event.get("unavailable") else "") + str(event.get("message") or "")] + + def outcome(self, raw, stream, last_file): + """The process reports a model call that broke the session off (a context overflow, a provider that stopped + serving) as an `error` event and exits 1: that session failed, with no answer to judge, as Pi's failed + request does; one the provider could not serve is run again first. A session ends with mini-swe-agent's + `exit` message: one that started and has none was cut off (the process killed), and failed too. (At the + Base Agent's own timeout or cancel CliAgent does not count it failed.)""" + notes, done = self.notices(raw), self.served(raw) + objects = _objects(raw)[0] + ended = any(e.get("type") == "message" and (e.get("message") or {}).get("role") == "exit" for e in objects) + broke = any(e.get("type") == "error" for e in objects) or (bool(objects) and not ended) + return {"reply": self.last_message(stream, last_file), "served": done, + "unavailable": bool(UNAVAILABLE.search(notes)), "refused": False, "failed": broke, + "failure": notes[-600:]} + + @classmethod + def last_message(cls, log, last=None): + """The runner's last-message file (never read through a link the session left): the submission, else the + last assistant message; else the same from the stream.""" + if last is not None and _regular(last): + return last.read_text(errors="replace") + reply = "" + for event in events(log): + message = event.get("message") or {} + if event.get("type") != "message": + continue + if message.get("role") == "exit" and message.get("submission"): + return str(message["submission"]) + if message.get("role") == "assistant" and str(message.get("content") or "").strip(): + reply = str(message["content"]) + return reply + + @classmethod + def parse(cls, log, model): + return parse_mini_swe_agent(log, model) + + @classmethod + def steps(cls, stream): + return mini_swe_agent_steps(stream) + + # Every CLI Base Agent, by each name make() takes for it. -CLIS: dict[str, type[CliAgent]] = {n: c for c in (Codex, Claude, Pi) for n in (c.backend, *c.aliases)} +CLIS: dict[str, type[CliAgent]] = {n: c for c in (Codex, Claude, Pi, MiniSweAgent) for n in (c.backend, *c.aliases)} def _this_call(home: Path, handle: str, reported, resumed: bool): diff --git a/hermit/agent/base/mini_swe_agent.py b/hermit/agent/base/mini_swe_agent.py new file mode 100644 index 00000000..847ed402 --- /dev/null +++ b/hermit/agent/base/mini_swe_agent.py @@ -0,0 +1,142 @@ +"""`mini-swe-agent`: mini-swe-agent (https://mini-swe-agent.com) as a Base Agent (docs/framework/agent-layers.md). + +mini-swe-agent's model runs bash commands and nothing else, in a loop its one YAML config describes. One `run` of +the Base Agent (hermit.agent.base.cli.MiniSweAgent) is one task of it in its own process +(hermit.agent.base.mini_swe_agent_run), run as mini-swe-agent's batch runner runs a task: through its Python API, +with its LocalEnvironment in `workspace`. That process writes the session as JSON events on its stdout, which +CliAgent keeps as the stream (`agent-mini-swe-agent.jsonl`) and reads into the reply, the trajectory +(hermit.agent.base.trajectory.mini_swe_agent_steps) and the usage; isolation, the home, the tool server and the clock +are CliAgent's, as for Codex and Claude Code. This module holds what the process and the deploy host share: the +quiet import of mini-swe-agent and the messages. + +The messages are Hermit's prompts (mini_swe_agent.yaml beside this module) over mini-swe-agent's own `mini` config +(`mini_textbased` for a text-based model class), then the `config` file's settings. The system message ends with +what the workspace's deployment gives (`hermit deploy --host mini-swe-agent` writes `mini-swe-agent.yaml`, whose +`hermit.context` is the adapter's context), the call's tools and the call's system instructions, under one heading +(`adapter_block`). A deployed workspace run with `mini -c mini-swe-agent.yaml` gets the same system message: the +deployment writes it into its system template (`baked`). + +The call's tools are one bash command, `hermit-tool ''`, which the process's environment +(hermit.agent.base.mini_swe_agent_env) answers through the tool server's JSON entrance. A command sees `env` and a +few basic variables (PATH, HOME, LANG, ...) only, not the process's environment, where the model's key is (on Linux +the process is non-dumpable, so a command cannot read it from /proc either); the key goes with each model request and +is never written to the stream. +""" +from __future__ import annotations + +import atexit +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import sys +import tempfile +import threading +from types import ModuleType +from typing import Sequence + +import yaml + +CONFIG_FILE = "mini-swe-agent.yaml" # a deployed workspace's config, as `mini -c` reads it +PROMPTS = Path(__file__).with_name("mini_swe_agent.yaml") +CONTEXT_HEADING = "# Instructions from the harness adapter" +PLACEHOLDER = "{{ adapter_context }}" +_PLACEHOLDER = re.compile(r"\{\{-?\s*adapter_context\s*-?\}\}") +_ENDRAW = re.compile(r"\{%[-+]?\s*endraw\s*[-+]?%\}") # every spelling Jinja ends a raw block with +STEP_LIMIT = 50 +INSTALL = ("mini-swe-agent is not installed: python -m pip install 'hermit-harness[mini-swe-agent]' " + "(from a checkout: python -m pip install -e '.[mini-swe-agent]')") +_LOADING = threading.Lock() + + +def load_minisweagent() -> ModuleType: + """Import mini-swe-agent, once per process, without the side effects it would have here: no banner, no + first-run wizard, and, unless MSWEA_GLOBAL_CONFIG_DIR is set, an empty config directory of its own (removed at + exit) rather than the user's, whose .env it would load into os.environ. litellm keeps its bundled price map + instead of fetching one: Hermit prices the tokens itself. Variables already set are kept.""" + with _LOADING: + if "minisweagent" not in sys.modules: + os.environ.setdefault("MSWEA_SILENT_STARTUP", "1") + os.environ.setdefault("MSWEA_CONFIGURED", "true") + os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") + if not os.environ.get("MSWEA_GLOBAL_CONFIG_DIR"): + own = tempfile.mkdtemp(prefix="hermit-mini-swe-agent-") + atexit.register(shutil.rmtree, own, ignore_errors=True) + os.environ["MSWEA_GLOBAL_CONFIG_DIR"] = own + try: + import minisweagent + except ImportError: + raise RuntimeError(INSTALL) from None + return minisweagent + + +# ---------- the messages: Hermit's prompts over mini-swe-agent's config, the adapter's block ---------- +def base_config(model_class: str) -> str: + """The mini-swe-agent config a model class goes with: `mini_textbased` for the text-based classes, else `mini`.""" + return "mini_textbased" if "textbased" in str(model_class) else "mini" + + +def render_config(base: str = "mini", *overrides: dict) -> dict: + """mini-swe-agent's own `base` config as the package ships it (its model and environment sections; never a + `mini.yaml` in the working directory or MSWEA_CONFIG_DIR), Hermit's prompts for it, then `overrides` (a user's + config), merged the way mini-swe-agent merges its -c files. The package is found, not imported: importing it + would change this process's environment (load_minisweagent), and a deployment only reads its files.""" + spec = importlib.util.find_spec("minisweagent") + if spec is None or not spec.submodule_search_locations: + raise RuntimeError(INSTALL) + prompts = yaml.safe_load(PROMPTS.read_text()) + if base not in prompts: + raise ValueError(f"no Hermit prompts for mini-swe-agent's {base!r} config; known: {sorted(prompts)}") + shipped = Path(list(spec.submodule_search_locations)[0]) / "config" / f"{base}.yaml" + return _merge(yaml.safe_load(shipped.read_text()), {"agent": prompts[base]}, *(o for o in overrides if o)) + + +def _merge(*configs: dict) -> dict: + """mini-swe-agent's recursive_merge: later configs win, mappings are merged key by key.""" + out: dict = {} + for config in configs: + for key, value in config.items(): + if isinstance(value, dict) and isinstance(out.get(key), dict): + out[key] = _merge(out[key], value) + else: + out[key] = _merge(value) if isinstance(value, dict) else value + return out + + +def tools_section(tools: Sequence[dict]) -> str: + """How to run the tools, and each one's name, description and argument schema; empty without tools.""" + if not tools: + return "" + lines = ["## Tools", "", + "Run a tool as a command of its own: `hermit-tool ''`, the arguments one JSON " + "object in single quotes. Its result is printed; `hermit-tool` alone lists the tools.", ""] + lines += [f"- `{t['name']}`: {t.get('description', '')} Arguments: " + f"`{json.dumps(t.get('input_schema') or {'type': 'object'}, ensure_ascii=False)}`" for t in tools] + return "\n".join(lines) + + +def adapter_block(*parts: str) -> str: + """What takes `{{ adapter_context }}`'s place: the non-empty parts (the adapter's context, the tools, a call's + system instructions) under one heading; empty when there are none.""" + parts = tuple(p.strip() for p in parts if p and p.strip()) + return f"\n{CONTEXT_HEADING}\n\n" + "\n\n".join(parts) + "\n" if parts else "" + + +def with_placeholder(template: str) -> str: + """A system template with `{{ adapter_context }}` in it: a template without one (a user's own) gets it at its + end.""" + return template if _PLACEHOLDER.search(template) else template.rstrip("\n") + "\n" + PLACEHOLDER + "\n" + + +def baked(template: str, block: str) -> str: + """The system template with the block in place of the placeholder as literal text (a Jinja raw block), for a + deployed config: mini-swe-agent renders it to what the Base Agent gets with `adapter_context=block`.""" + text = _ENDRAW.sub(lambda m: "{% endraw %}{{ " + json.dumps(m[0]) + " }}{% raw %}", block) + raw = "{% raw %}" + text + "{% endraw %}" + return _PLACEHOLDER.sub(lambda _: raw, with_placeholder(template)) + + +__all__ = ["CONFIG_FILE", "STEP_LIMIT", "adapter_block", "base_config", "baked", "load_minisweagent", + "render_config", "tools_section", "with_placeholder"] diff --git a/hermit/agent/base/mini_swe_agent.yaml b/hermit/agent/base/mini_swe_agent.yaml new file mode 100644 index 00000000..683439fa --- /dev/null +++ b/hermit/agent/base/mini_swe_agent.yaml @@ -0,0 +1,210 @@ +# The agent prompts of the mini-swe-agent Base Agent and of `hermit deploy --host mini-swe-agent`, one set per +# mini-swe-agent base config: `mini` (the bash tool, mini-swe-agent's default) and `mini_textbased` (one +# ```mswea_bash_command block per response). Each is that config's agent section from mini-swe-agent 2.4.6 with +# two changes: the task is any task rather than an issue in a repository, and the agent submits its answer (the +# output the judge scores) as mini-swe-agent's swebench config submits a patch. `{{ adapter_context }}` is where the +# adapter's context goes; the model and environment sections stay mini-swe-agent's own (hermit/agent/base/mini_swe_agent.py). +mini: + system_template: | + You are a helpful assistant that can interact with a computer. + {{ adapter_context }} + instance_template: | + Please solve this task: {{task}} + + You can execute bash commands and edit files to complete it. + + ## Recommended Workflow + + This workflow should be done step-by-step so that you can iterate on your work and any possible problems. + + 1. Read the task and look at the files in the working directory + 2. Work toward the answer or the requested changes, checking intermediate results + 3. Verify your result + 4. Write your final answer (what the task asks you to return) to a file, for example `answer.txt` + 5. Submit it and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT && cat answer.txt`. + Issue it as a command of its own once the file is written. After this command, you cannot continue working on this task. + + ## Command Execution Rules + + You are operating in an environment where + + 1. You issue at least one command + 2. The system executes the command(s) in a subshell + 3. You see the result(s) + 4. You write your next command(s) + + Each response should include: + + 1. **Reasoning text** where you explain your analysis and plan + 2. At least one tool call with your command + + **CRITICAL REQUIREMENTS:** + + - Your response SHOULD include reasoning text explaining what you're doing + - Your response MUST include AT LEAST ONE bash tool call + - Directory or environment variable changes are not persistent. Every action is executed in a new subshell. + - However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files + - Submit your answer and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT && cat answer.txt`. + Issue it as a command of its own once the file is written. After this command, you cannot continue working on this task. + + Example of a CORRECT response: + + I need to see what is in the working directory first. Let me list its files. + + [Makes bash tool call with {"command": "ls -la"} as arguments] + + + + {{system}} {{release}} {{version}} {{machine}} + + + ## Useful command examples + + ### Create a new file: + + ```bash + cat <<'EOF' > newfile.py + import numpy as np + hello = "world" + print(hello) + EOF + ``` + + ### Edit files with sed: + + {%- if system == "Darwin" -%} + + You are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`. + + {%- endif -%} + + ```bash + # Replace all occurrences + sed -i 's/old_string/new_string/g' filename.py + + # Replace only first occurrence + sed -i 's/old_string/new_string/' filename.py + + # Replace first occurrence on line 1 + sed -i '1s/old_string/new_string/' filename.py + + # Replace all occurrences in lines 1-10 + sed -i '1,10s/old_string/new_string/g' filename.py + ``` + + ### View file content: + + ```bash + # View specific lines with numbers + nl -ba filename.py | sed -n '10,20p' + ``` + + ### Any other command you want to run + + ```bash + anything + ``` + +mini_textbased: + system_template: | + You are a helpful assistant that can interact with a computer. + + Your response must contain exactly ONE bash code block with ONE command (or commands connected with && or ||). + Include a THOUGHT section before your command where you explain your reasoning process. + Format your response as shown in . + + + Your reasoning and analysis here. Explain why you want to perform the action. + + ```mswea_bash_command + your_command_here + ``` + + + Failure to follow these rules will cause your response to be rejected. + {{ adapter_context }} + instance_template: | + Please solve this task: {{task}} + + You can execute bash commands and edit files to complete it. + + ## Recommended Workflow + + This workflow should be done step-by-step so that you can iterate on your work and any possible problems. + + 1. Read the task and look at the files in the working directory + 2. Work toward the answer or the requested changes, checking intermediate results + 3. Verify your result + 4. Write your final answer (what the task asks you to return) to a file, for example `answer.txt` + 5. Submit it and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT && cat answer.txt`. + Issue it as a command of its own once the file is written. After this command, you cannot continue working on this task. + + ## Important Rules + + 1. Every response must contain exactly one action + 2. The action must be enclosed in triple backticks + 3. Directory or environment variable changes are not persistent. Every action is executed in a new subshell. + However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files + + + {{system}} {{release}} {{version}} {{machine}} + + + ## Formatting your response + + Here is an example of a correct response: + + + THOUGHT: I need to see what is in the working directory first. Let me list its files. + + ```mswea_bash_command + ls -la + ``` + + + ## Useful command examples + + ### Create a new file: + + ```mswea_bash_command + cat <<'EOF' > newfile.py + import numpy as np + hello = "world" + print(hello) + EOF + ``` + + ### Edit files with sed: + + {%- if system == "Darwin" -%} + + You are on MacOS. For all the below examples, you need to use `sed -i ''` instead of `sed -i`. + + {%- endif -%} + + ```mswea_bash_command + # Replace all occurrences + sed -i 's/old_string/new_string/g' filename.py + + # Replace only first occurrence + sed -i 's/old_string/new_string/' filename.py + + # Replace first occurrence on line 1 + sed -i '1s/old_string/new_string/' filename.py + + # Replace all occurrences in lines 1-10 + sed -i '1,10s/old_string/new_string/g' filename.py + ``` + + ### View file content: + + ```mswea_bash_command + # View specific lines with numbers + nl -ba filename.py | sed -n '10,20p' + ``` + + ### Any other command you want to run + + ```mswea_bash_command + anything + ``` diff --git a/hermit/agent/base/mini_swe_agent_env.py b/hermit/agent/base/mini_swe_agent_env.py new file mode 100644 index 00000000..1d302dcb --- /dev/null +++ b/hermit/agent/base/mini_swe_agent_env.py @@ -0,0 +1,168 @@ +"""HermitEnvironment: mini-swe-agent's local environment with the call's tools as one bash command. + +mini-swe-agent's model runs bash commands and nothing else, so a tool is a command of its own: + + hermit-tool '' # the result is printed; an error exits 1 + hermit-tool # the tools, one per line + +The environment answers it in mini-swe-agent's process and runs every other command as LocalEnvironment does. Here +the tools are the call's, served by the Base Agent's tool server (its JSON entrance, `tools_url`); +hermit.hosts.mini_swe_agent_env serves a deployed workspace's adapter tools the same way through a Session. + +With `inherit_env: false` (the Base Agent) a command sees `env`, the basic variables in BASIC_ENV and `extra_env` +only, never the process's environment, where the model's key is. `extra_env` is not part of the config, which +mini-swe-agent writes into the trajectory. This module imports mini-swe-agent: import it in mini-swe-agent's process. +""" +from __future__ import annotations + +import json +import os +import shlex +import signal +import subprocess +from typing import Any +import urllib.error +import urllib.request + +from minisweagent.environments.local import LocalEnvironment, LocalEnvironmentConfig + +TOOL_COMMAND = "hermit-tool" +TOOLS_ENV = "HERMIT_TOOLS_URL" # the tool server's JSON entrance, as the Base Agent passes it +BASIC_ENV = ("PATH", "HOME", "USER", "LOGNAME", "SHELL", "LANG", "LANGUAGE", "LC_ALL", "LC_CTYPE", "TERM", "TMPDIR", + "TZ") +CALL_TIMEOUT_S = 600 +RUNNING: set[int] = set() # the process groups of the commands running now (stop_commands) + + +class HermitEnvironmentConfig(LocalEnvironmentConfig): + tools_url: str = "" + """The tool server's JSON entrance (GET /tools, POST /call/); empty: the HERMIT_TOOLS_URL + variable, else no tools.""" + inherit_env: bool = True + """False: commands get `env`, BASIC_ENV and `extra_env` only, not this process's environment.""" + + +class HermitEnvironment(LocalEnvironment): + def __init__(self, *, extra_env: dict[str, str] | None = None, config_class: type = HermitEnvironmentConfig, + **kwargs): + super().__init__(config_class=config_class, **kwargs) + self.extra_env = dict(extra_env or {}) + + def execute(self, action: dict, cwd: str = "", *, timeout: int | None = None) -> dict[str, Any]: + command = action.get("command", "") + try: + words = shlex.split(command) + except ValueError: + words = [] + if words and words[0] == TOOL_COMMAND: + return self._tool(words[1:]) + if self.config.inherit_env: + return super().execute(action, cwd, timeout=timeout) + env = {k: os.environ[k] for k in BASIC_ENV if k in os.environ} | self.config.env | self.extra_env + try: + result = _run(command, cwd or self.config.cwd or os.getcwd(), env, timeout or self.config.timeout) + output = {"output": result.stdout, "returncode": result.returncode, "exception_info": ""} + except Exception as e: # as LocalEnvironment.execute reports a command that failed to run + raw = getattr(e, "output", None) + raw = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else (raw or "") + output = {"output": raw, "returncode": -1, + "exception_info": f"An error occurred while executing the command: {e}", + "extra": {"exception_type": type(e).__name__, "exception": str(e)}} + self._check_finished(output) + return output + + # ---------- the tools: the tool server's here, a Session's in a deployed workspace ---------- + def tool_list(self) -> list[dict]: + """The tools: name, description and input schema of each.""" + url = self._url() + return _request(f"{url}/tools") if url else [] + + def call_tool(self, name: str, arguments: dict) -> str: + """One call's result as the text the model reads; raises on an error.""" + url = self._url() + if not url: + raise ValueError(f"Unknown tool: {name}") + answer = _request(f"{url}/call/{name}", arguments) + if "error" in answer: + raise RuntimeError(answer["error"]) + return text(answer.get("result")) + + def _url(self) -> str: + return (self.config.tools_url or os.environ.get(TOOLS_ENV, "")).rstrip("/") + + def _tool(self, words: list[str]) -> dict[str, Any]: + """A `hermit-tool` command's output: the tool list, the tool's result, or the error (exit 1).""" + try: + tools = self.tool_list() + except Exception as exc: + return _out(f"Error: {type(exc).__name__}: {exc}", 1) + if not tools: + return _out("Error: no tools are available here", 1) + if not words: + return _out("\n".join(f"{t['name']}: {t['description']}" for t in tools), 0) + if len(words) > 2: + return _out(f"Error: run {TOOL_COMMAND} as a command of its own: {TOOL_COMMAND} '', " + "with the arguments as one JSON object in single quotes", 1) + name = words[0] + try: + if name not in {t["name"] for t in tools}: + raise ValueError(f"Unknown tool: {name}") + arguments = json.loads(words[1]) if len(words) > 1 else {} + if not isinstance(arguments, dict): + raise ValueError("tool arguments must be a JSON object") + return _out(self.call_tool(name, arguments), 0) + except Exception as exc: + return _out(f"Error: {type(exc).__name__}: {exc}", 1) + + +def _run(command: str, cwd: str, env: dict[str, str], timeout: int) -> subprocess.CompletedProcess[str]: + """mini-swe-agent's own _run (a command in a process group of its own, the group killed at its timeout), with + the group recorded while it runs, so that stop_commands reaches it.""" + process = subprocess.Popen(command, shell=True, text=True, cwd=cwd, env=env, encoding="utf-8", errors="replace", + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, start_new_session=True) + RUNNING.add(process.pid) + try: + stdout, _ = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + stdout, _ = process.communicate() + raise subprocess.TimeoutExpired(command, timeout, output=stdout) + finally: + RUNNING.discard(process.pid) + return subprocess.CompletedProcess(command, process.returncode, stdout=stdout) + + +def stop_commands() -> None: + """Kill the commands running now: each runs in a session of its own, which a kill of this process's group does + not reach (on macOS nothing else finds them: the Base Agent's stop walks /proc).""" + for pid in list(RUNNING): + try: + os.killpg(pid, signal.SIGKILL) + except (ProcessLookupError, PermissionError): + pass + + +def text(result: Any) -> str: + """A tool result as the text a model reads: a result's own `text` field, else its JSON (as Session.text).""" + if isinstance(result, dict) and isinstance(result.get("text"), str): + return result["text"] + return result if isinstance(result, str) else json.dumps(result, ensure_ascii=False, default=str) + + +def _request(url: str, body: dict | None = None) -> Any: + data = None if body is None else json.dumps(body).encode() + request = urllib.request.Request(url, data=data, method="GET" if body is None else "POST", + headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(request, timeout=CALL_TIMEOUT_S) as response: + return json.loads(response.read() or b"null") + except urllib.error.HTTPError as error: # an unknown tool is a 404 with a JSON error + return json.loads(error.read() or b"{}") + + +def _out(text: str, returncode: int) -> dict[str, Any]: + return {"output": text if text.endswith("\n") or not text else text + "\n", "returncode": returncode, + "exception_info": ""} + + +__all__ = ["BASIC_ENV", "HermitEnvironment", "HermitEnvironmentConfig", "TOOL_COMMAND", "TOOLS_ENV", "stop_commands"] diff --git a/hermit/agent/base/mini_swe_agent_run.py b/hermit/agent/base/mini_swe_agent_run.py new file mode 100644 index 00000000..2f824d91 --- /dev/null +++ b/hermit/agent/base/mini_swe_agent_run.py @@ -0,0 +1,275 @@ +"""One mini-swe-agent task, run as the `mini-swe-agent` Base Agent's process (hermit.agent.base.mini_swe_agent). + + python -m hermit.agent.base.mini_swe_agent_run --settings /mini-swe-agent.json --cwd \ + --last /last_message.txt --stdin < prompt + +The task runs as mini-swe-agent's batch runner runs one: a model, an environment and a DefaultAgent from its Python +API, the config Hermit's prompts over mini-swe-agent's own (hermit.agent.base.mini_swe_agent.render_config) and the +settings' `config` over them. Every message is written to stdout as one JSON event as it is added; the reply (the +submission, else the last assistant message) goes to `--last`. Events: + + {"type": "session.started", "session": , "model": } + {"type": "message", "message": {"role", "content", "actions", "returncode", "output", "exit_status", + "submission", ...}, "usage": {"input_tokens", "cached_input_tokens", + "output_tokens"}} # usage: on a model response + {"type": "error", "message": ": ", "unavailable": true} # then the process exits 1; + # `unavailable`: the provider could not serve (after mini-swe-agent's retries) + +A setting that fails before the session (a config that names the model or the environment class), or a first model +call that fails for a setting (a wrong key, a model that does not exist), is reported on stderr, with no event, and +the process exits 2: the Base Agent's caller sees a session that never started, not a wrong answer. Until the model +has answered, the events are held back for that reason (Stream). At SIGTERM (the Base Agent's timeout) the running +commands are killed too. On Linux the process makes itself non-dumpable, so the commands it runs (the same user) +cannot read its environment, where the model's key is. + +The model's key is read from the binding's variable and passed with each request; it is in no event, and the +commands' environment holds `env` (the variables `env_names` names) and a few basic ones only. +""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import signal +import sys +from typing import Any, Callable +import uuid + +import yaml + +from hermit.agent.base.mini_swe_agent import (CONFIG_FILE, STEP_LIMIT, adapter_block, base_config, + load_minisweagent, render_config, tools_section, with_placeholder) +from hermit.agent.model.backends import PLACEHOLDER_KEY + +CALL_TIMEOUT_S = 600 # one model request, at most; the session's own clock is the Base Agent's + + +def event_of(message: dict) -> dict: + """A mini-swe-agent message as a stream event: its chat fields, the commands it asked for, an observation's + output and exit code, an exit's status and submission, and a response's token counts; not its raw response.""" + extra = message.get("extra") or {} + content = message.get("content") + if isinstance(content, list): # a multimodal message: its text parts + content = "\n".join(str(p.get("text") or "") for p in content if isinstance(p, dict)) + out: dict[str, Any] = {"role": message.get("role"), "content": "" if content is None else str(content)} + for key in ("tool_calls", "tool_call_id"): + if message.get(key) is not None: + out[key] = message[key] + if extra.get("actions"): + out["actions"] = [str(a.get("command", "")) for a in extra["actions"] if isinstance(a, dict)] + if "returncode" in extra: + out["returncode"] = extra["returncode"] + out["output"] = str(extra.get("raw_output") or "") + for key in ("exit_status", "submission"): + if key in extra: + out[key] = extra[key] + event: dict[str, Any] = {"type": "message", "message": out} + response = extra.get("response") + usage = (response.get("usage") if isinstance(response, dict) else None) or {} + if usage: + event["usage"] = {"input_tokens": int(usage.get("prompt_tokens") or 0), + "cached_input_tokens": int((usage.get("prompt_tokens_details") or {}).get("cached_tokens") + or 0), + "output_tokens": int(usage.get("completion_tokens") or 0)} + return event + + +def _emit(event: dict) -> None: + sys.stdout.write(json.dumps(event, ensure_ascii=False, default=str) + "\n") + sys.stdout.flush() + + +def bound_model(model_class: type, binding: dict | None) -> type: + """`model_class` calling the binding's endpoint with its key, both passed with each request (the model's config, + which mini-swe-agent writes into its trajectory, holds neither), each request bounded by CALL_TIMEOUT_S.""" + settings: dict[str, Any] = {"timeout": CALL_TIMEOUT_S} + if binding: + settings["api_base"] = binding["base_url"] + settings["api_key"] = os.environ.get(binding.get("api_key_env") or "", "") or PLACEHOLDER_KEY + + import litellm + + class Bound(model_class): + # a 400 is the request's: sent again it fails again (a wrong key on DeepSeek is one), so it is not retried + abort_exceptions = [*getattr(model_class, "abort_exceptions", []), litellm.exceptions.BadRequestError] + + def _query(self, messages, **kwargs): + return super()._query(messages, **{**settings, **kwargs}) + + Bound.__name__, Bound.__qualname__, Bound.__module__ = (model_class.__name__, model_class.__qualname__, + model_class.__module__) + return Bound + + +def build(settings: dict, cwd: str, emit: Callable[[dict], None]): + """(the agent, the adapter's block) for one task.""" + load_minisweagent() + import litellm + from minisweagent.agents.default import DefaultAgent + from minisweagent.models import get_model_class + from hermit.agent.base.mini_swe_agent_env import HermitEnvironment + + litellm.suppress_debug_info = True # its help text would go to stdout, between the events + model_class = str(settings.get("model_class") or "litellm") + user = yaml.safe_load(Path(settings["config"]).read_text()) if settings.get("config") else None + user = user or {} + given = [f"model.{k}" for k in ("model_name", "model_class") if k in (user.get("model") or {})] + given += ["environment.environment_class"] if "environment_class" in (user.get("environment") or {}) else [] + if given: + raise ValueError(f"mini-swe-agent's config sets {' and '.join(given)}: the model is the Base Agent's model " + "and model_class, and the environment is Hermit's local one") + config = render_config(base_config(model_class), user) + + deployed = Path(cwd) / CONFIG_FILE # the workspace's deployment, as `mini -c` would read it + context = "" + if deployed.is_file() and not deployed.is_symlink(): + context = str(((yaml.safe_load(deployed.read_text()) or {}).get("hermit") or {}).get("context") or "") + env_config = {k: v for k, v in (config.get("environment") or {}).items() if k != "environment_class"} + extra_env = {n: os.environ[n] for n in settings.get("env_names") or [] if n in os.environ} + env = HermitEnvironment(extra_env=extra_env, **{**env_config, "cwd": cwd, "inherit_env": False}) + block = adapter_block(context, tools_section(env.tool_list()), str(settings.get("system") or "")) + + binding = settings.get("binding") + model_config = dict(config.get("model") or {}) + model_kwargs = dict(model_config.pop("model_kwargs", None) or {}) + if binding and binding.get("extra_body"): + model_kwargs["extra_body"] = {**binding["extra_body"], **(model_kwargs.get("extra_body") or {})} + if settings.get("effort"): + model_kwargs.setdefault("reasoning_effort", settings["effort"]) + name = binding["litellm"] if binding else settings.get("model") + if not name: + raise ValueError("mini-swe-agent needs a model: a / reference or a litellm model id") + model = bound_model(get_model_class(name, model_class), binding)( + **{**model_config, "model_name": name, "model_kwargs": model_kwargs, "cost_tracking": "ignore_errors"}) + + class Streaming(DefaultAgent): + def add_messages(self, *messages: dict) -> list[dict]: + added = super().add_messages(*messages) + for message in messages: + emit(event_of(message)) + return added + + agent_config = {k: v for k, v in (config.get("agent") or {}).items() if k != "agent_class"} + agent_config["system_template"] = with_placeholder(agent_config["system_template"]) + step_limit = (user.get("agent") or {}).get("step_limit") # 0 is mini-swe-agent's "no limit" + step_limit = settings.get("step_limit") if step_limit is None else step_limit + agent_config["step_limit"] = int(STEP_LIMIT if step_limit is None else step_limit) + agent_config["cost_limit"] = 0 # the cost is Hermit's to count (cost tracking is off) + agent_config["output_path"] = None + return Streaming(model, env, **agent_config), block + + +def reply(messages: list[dict]) -> str: + """The submission (the output of the command that submitted it, its end of line left out), else the last + assistant message.""" + if messages and messages[-1].get("role") == "exit" and (messages[-1].get("extra") or {}).get("submission"): + return str(messages[-1]["extra"]["submission"]).strip() + for message in reversed(messages): + content = message.get("content") + if message.get("role") == "assistant" and isinstance(content, str) and content.strip(): + return content + return "" + + +def _setup_error(exc: Exception) -> bool: + """Whether the first model call failed for a setting every task shares: the key, the model id, a parameter. The + first request holds only the system message and the task, so a 400 then is the settings' (some providers answer + a wrong key with one), unless it is the task's size or content.""" + import litellm + e = litellm.exceptions + if isinstance(exc, (e.ContextWindowExceededError, e.ContentPolicyViolationError)): + return False + return isinstance(exc, (e.AuthenticationError, e.PermissionDeniedError, e.NotFoundError, e.BadRequestError)) + + +class Stream: + """The events on stdout. Until the model has answered once they are held back: a call that fails then for a + setting every task shares (a wrong key, a model that does not exist) is reported as a session that never started + (stderr, no event), not as a task without an answer.""" + + def __init__(self, started: dict): + self.started, self.held = started, [] + + @property + def answered(self) -> bool: + return self.held is None + + def __call__(self, event: dict) -> None: + if self.held is None: + _emit(event) + return + self.held.append(event) + message = event.get("message") or {} + if message.get("role") == "assistant" or event.get("usage"): # a response, a format error's included + self.flush() + + def flush(self) -> None: + if self.held is not None: + for event in [self.started, *self.held]: + _emit(event) + self.held = None + + +def _stop(signum, frame) -> None: + """The Base Agent stops this process at its timeout: the commands running in sessions of their own go too.""" + from hermit.agent.base.mini_swe_agent_env import stop_commands + stop_commands() + os._exit(128 + signum) + + +def _unavailable(exc: Exception) -> bool: + """Whether the provider could not serve (a rate limit, an outage, no connection), not the request was wrong.""" + import litellm + e = litellm.exceptions + return isinstance(exc, (e.RateLimitError, e.ServiceUnavailableError, e.APIConnectionError, e.InternalServerError, + e.BadGatewayError, e.Timeout)) + + +def _not_dumpable() -> None: + """On Linux, this process's memory and /proc//environ out of reach of the commands it runs (the same + user's processes), as Codex keeps its own: the model's key is in this process's environment.""" + if sys.platform.startswith("linux"): + import ctypes + ctypes.CDLL(None, use_errno=True).prctl(4, 0, 0, 0, 0) # PR_SET_DUMPABLE, 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--settings", type=Path, required=True) + parser.add_argument("--cwd", required=True) + parser.add_argument("--last", type=Path, required=True) + parser.add_argument("--prompt") + parser.add_argument("--stdin", action="store_true") + args = parser.parse_args(argv) + _not_dumpable() + signal.signal(signal.SIGTERM, _stop) + prompt = sys.stdin.read() if args.stdin else (args.prompt or "") + settings = json.loads(args.settings.read_text()) + args.last.write_text("", encoding="utf-8") + stream = Stream({"type": "session.started", "session": uuid.uuid4().hex, + "model": (settings.get("binding") or {}).get("litellm") or settings.get("model")}) + try: + agent, block = build(settings, args.cwd, stream) + except Exception as exc: # a setting that fails every task: no session starts, nothing is an event + print(f"hermit mini-swe-agent: {type(exc).__name__}: {exc}", file=sys.stderr, flush=True) + return 2 + try: + agent.run(prompt, adapter_context=block) + except Exception as exc: # the run's exit message is in the stream (DefaultAgent adds it) + if not stream.answered and _setup_error(exc): + print(f"hermit mini-swe-agent: {type(exc).__name__}: {exc}", file=sys.stderr, flush=True) + return 2 + stream.flush() + event = {"type": "error", "message": f"{type(exc).__name__}: {exc}"} + if _unavailable(exc): + event["unavailable"] = True + _emit(event) + return 1 # no answer: the last message of a run that broke off is none + stream.flush() + args.last.write_text(reply(agent.messages), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hermit/agent/base/records.py b/hermit/agent/base/records.py index 225360d0..1b070e16 100644 --- a/hermit/agent/base/records.py +++ b/hermit/agent/base/records.py @@ -124,6 +124,18 @@ def parse_pi(log: Path, model: str) -> tuple[str | None, list[Usage]]: return _parse(log, model, _pi_event) +def _mini_swe_agent_event(event: dict, usage: Usage, session): + if event.get("type") == "session.started": + session = event.get("session") or session + if event.get("type") == "message" and isinstance(event.get("usage"), dict): + u = event["usage"] + usage.input_tokens += int(u.get("input_tokens", 0)) + usage.cached_input_tokens += int(u.get("cached_input_tokens", 0)) + usage.output_tokens += int(u.get("output_tokens", 0)) + usage.calls += 1 + return session + + def parse_codex(log: Path, model: str) -> tuple[str | None, list[Usage]]: """Codex's `exec --json` events: the thread id, and each completed turn's usage.""" return _parse(log, model, _codex_event) @@ -134,6 +146,14 @@ def parse_claude(log: Path, model: str) -> tuple[str | None, list[Usage]]: return _parse(log, model, _claude_event) +def parse_mini_swe_agent(log: Path, model: str) -> tuple[str | None, list[Usage]]: + """mini-swe-agent's events (hermit.agent.base.mini_swe_agent_run): the session id, and each model response's + usage (a response that broke the format included: it was billed).""" + return _parse(log, model, _mini_swe_agent_event) + + def parse_log(log: Path, backend: str, model: str) -> tuple[str | None, list[Usage]]: - """Session handle and usage from the agent's JSON event stream: Codex's, else Claude Code's.""" - return (parse_codex if backend == "codex" else parse_claude)(log, model) + """Session handle and usage from the agent's JSON event stream: Codex's, Pi's, mini-swe-agent's, else Claude + Code's.""" + return {"codex": parse_codex, "pi": parse_pi, "mini-swe-agent": parse_mini_swe_agent}.get(backend, parse_claude)( + log, model) diff --git a/hermit/agent/base/trajectory.py b/hermit/agent/base/trajectory.py index 8cb731df..b33039af 100644 --- a/hermit/agent/base/trajectory.py +++ b/hermit/agent/base/trajectory.py @@ -9,9 +9,9 @@ text(steps, message=1200, output=600) # one line per step, each field clipped from its start The streams read: Codex's `codex exec --json` events (`agent-codex.jsonl`), Claude Code's stream-json -(`agent-claude.jsonl`) and Pi's `--mode json` events (`agent-pi.jsonl`), also under another name below a log -directory (Harbor's `agent/codex.txt`). A normalised -message list (`RunOutput.messages`) is the pipeline's own rendering and is left to each Evolver. +(`agent-claude.jsonl`), Pi's `--mode json` events (`agent-pi.jsonl`) and mini-swe-agent's events +(`agent-mini-swe-agent.jsonl`), also under another name below a log directory (Harbor's `agent/codex.txt`). A +normalised message list (`RunOutput.messages`) is the pipeline's own rendering and is left to each Evolver. """ from __future__ import annotations @@ -22,7 +22,10 @@ CODEX_STREAM = "agent-codex.jsonl" CLAUDE_STREAM = "agent-claude.jsonl" PI_STREAM = "agent-pi.jsonl" -STREAM_HOSTS = {CODEX_STREAM: "codex", CLAUDE_STREAM: "claude", PI_STREAM: "pi"} # the file a host's log keeps +MINI_SWE_AGENT_STREAM = "agent-mini-swe-agent.jsonl" +# the file a host's log keeps its events under +STREAM_HOSTS = {CODEX_STREAM: "codex", CLAUDE_STREAM: "claude", PI_STREAM: "pi", + MINI_SWE_AGENT_STREAM: "mini-swe-agent"} STREAMS = tuple(STREAM_HOSTS) OTHER_STREAMS = ("agent/codex.txt", "codex.txt") # a Harbor trial's Codex events: from the trial or its agent/ @@ -213,10 +216,51 @@ def pi_steps(stream: list[dict]) -> list[Step]: return steps +def mini_swe_agent_steps(stream: list[dict]) -> list[Step]: + """mini-swe-agent's events (hermit.agent.base.mini_swe_agent_run): each assistant message's text and its + commands, each command with the output and exit code of the observation that answers it, a message the agent + was sent besides (a format error), an exit other than a submission and the process's errors as errors. The system + message and the task are left out, as the CLIs' streams leave out their prompt.""" + steps, pending, prompts = [], [], 0 + for event in stream: + kind, message = event.get("type"), event.get("message") or {} + if kind == "error": + text = str(event.get("message") or "") + if not (steps and steps[-1].kind == "error" and steps[-1].text == text): # mini-swe-agent's exit, again + steps.append(Step("error", text)) + continue + if kind != "message": + continue + role, content = message.get("role"), str(message.get("content") or "") + if role == "assistant": + if content.strip(): + steps.append(Step("message", content)) + for command in message.get("actions") or []: + step = Step("command", command=str(command)) + steps.append(step) + pending.append(step) + elif role in ("tool", "user") and "returncode" in message and pending: + step = pending.pop(0) + step.exit_code, step.output = message.get("returncode"), str(message.get("output") or "") + elif role == "user": + prompts += 1 + if prompts > 1 and content.strip(): # the first is the task + steps.append(Step("message", content, role="user")) + elif role == "exit" and message.get("exit_status") == "Submitted" and pending: + # the command that submitted (the response's others never ran): its output is the submission + step = next((p for p in pending if "COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT" in p.command), pending[-1]) + step.exit_code, step.output = 0, str(message.get("submission") or "") + pending.clear() + elif role == "exit" and message.get("exit_status") != "Submitted": + steps.append(Step("error", f"{message.get('exit_status')}: {content}".rstrip(": "))) + return steps + + # Each host's stream: the reader of its events, and the event types that mark it (tried in this order when a stream # is found by its events). A host's stream is added here and in STREAM_HOSTS. -READERS = {"codex": codex_steps, "claude": claude_steps, "pi": pi_steps} +READERS = {"codex": codex_steps, "claude": claude_steps, "pi": pi_steps, "mini-swe-agent": mini_swe_agent_steps} MARKS = (("codex", {"thread.started", "turn.started", "turn.completed", "turn.failed", "item.completed"}), + ("mini-swe-agent", {"session.started"}), ("claude", {"system", "assistant", "user", "result"}), ("pi", {"agent_start", "agent_settled", "message_end", "tool_execution_end"})) diff --git a/hermit/agent/model/backends.py b/hermit/agent/model/backends.py index 78b02ce8..af8c8295 100644 --- a/hermit/agent/model/backends.py +++ b/hermit/agent/model/backends.py @@ -61,7 +61,8 @@ def user_file() -> Path: "context_window", "developer_role", "models", "description"} DEVELOPER_ROLES = ("keep", "system") # What each consumer speaks: the dialect a backend must serve for it. -NEEDS = {"chat": "chat", "react": "chat", "llm": "chat", "codex": "responses", "claude": "messages", "pi": "chat"} +NEEDS = {"chat": "chat", "react": "chat", "llm": "chat", "codex": "responses", "claude": "messages", "pi": "chat", + "mini-swe-agent": "chat"} # Pi speaks several dialects: the one it is given is the first the backend serves (Binding.pi_dialect). PI_APIS = {"chat": "openai-completions", "messages": "anthropic-messages", "responses": "openai-responses"} # The variable litellm reads the key from, per route prefix (the examples' CL-Bench host calls through litellm). @@ -396,11 +397,11 @@ def _spec_of(value, binding: Binding) -> dict: def bind_agent_params(params: dict, kind: str, backends: dict[str, Backend], default=None) -> dict: - """The params of a Codex (`kind` codex), Claude Code (`kind` claude) or Pi (`kind` pi) agent or Evolver with its - model bound: `model` becomes the id the endpoint expects, the provider settings go into `args` (Codex), `env` - (Claude Code) ahead of the user's own, or `provider` (Pi's models.json entry), `price` is filled from the - backend, and `binding` records the whole. A `model` that is no reference (the CLI's own provider) is left - alone.""" + """The params of a Codex (`kind` codex), Claude Code (`kind` claude), Pi (`kind` pi) or mini-swe-agent (`kind` + mini-swe-agent) agent or Evolver with its model bound: `model` becomes the id the endpoint expects, the provider + settings go into `args` (Codex), `env` (Claude Code) ahead of the user's own, or `provider` (Pi's models.json + entry; mini-swe-agent reads them from `binding`), `price` is filled from the backend, and `binding` records the + whole. A `model` that is no reference (the CLI's own provider) is left alone.""" binding = as_binding(params.get("model") or default, backends) if binding is None: return params @@ -412,7 +413,7 @@ def bind_agent_params(params: dict, kind: str, backends: dict[str, Backend], def out["env"] = {**binding.claude_env(), **(params.get("env") or {})} elif kind == "pi": out["provider"] = params.get("provider") or binding.pi_provider() - else: + elif kind != "mini-swe-agent": raise ValueError(f"kind must be one of {CLI_KINDS}, not {kind!r}") if not out.get("price"): out["price"] = binding.price @@ -422,8 +423,9 @@ def bind_agent_params(params: dict, kind: str, backends: dict[str, Backend], def # ---- the run config ------------------------------------------------------------------------------------------- -CLI_AGENTS = {"codex": "codex", "claude-code": "claude", "claude": "claude", "pi": "pi", "cli": None} -CLI_KINDS = ("codex", "claude", "pi") # the CLI Base Agents a model is bound for (bind_agent_params) +CLI_AGENTS = {"codex": "codex", "claude-code": "claude", "claude": "claude", "pi": "pi", + "mini-swe-agent": "mini-swe-agent", "cli": None} +CLI_KINDS = ("codex", "claude", "pi", "mini-swe-agent") # the CLI Base Agents a model is bound for (bind_agent_params) def needs(kind: str, binding: Binding) -> str: @@ -432,9 +434,10 @@ def needs(kind: str, binding: Binding) -> str: def _cli_kind(agent: dict, params: dict) -> str: - """codex, claude or pi for an acting agent that is a CLI (by its name, `cli`'s backend, or a task's own agent's - agent.base), else "".""" - names = {"codex": "codex", "claude": "claude", "claude-code": "claude", "pi": "pi"} + """codex, claude, pi or mini-swe-agent for an acting agent that is a CLI (by its name, `cli`'s backend, or a task's + own agent's agent.base), else "".""" + names = {"codex": "codex", "claude": "claude", "claude-code": "claude", "pi": "pi", + "mini-swe-agent": "mini-swe-agent"} kind = CLI_AGENTS.get(str(agent.get("name") or ""), "") if kind is None: # `cli`: the backend is a param return names.get(params.get("backend", "codex"), "") diff --git a/hermit/cli.py b/hermit/cli.py index d2e9390f..39c7bcbc 100644 --- a/hermit/cli.py +++ b/hermit/cli.py @@ -30,11 +30,12 @@ def main(argv=None) -> int: p.add_argument("--adapter", required=True, help="final | root | none | a tag | a commit | a local path") if name == "check": p.add_argument("--no-probe", action="store_true", help="skip running one task through the pipeline") - dep = sub.add_parser("deploy", help="deploy an adapter's exports into a Codex, Claude Code or Pi workspace") + dep = sub.add_parser("deploy", help="deploy an adapter's exports into a Codex, Claude Code, Pi or mini-swe-agent " + "workspace") dep.add_argument("--adapter", type=Path, required=True) dep.add_argument("--workspace", "--output", dest="workspace", type=Path, required=True, help="the project directory the agent will work in (--output is the older spelling)") - dep.add_argument("--host", choices=["codex", "claude-code", "pi"], default="codex") + dep.add_argument("--host", choices=["codex", "claude-code", "pi", "mini-swe-agent"], default="codex") dep.add_argument("--python", default=sys.executable) dep.add_argument("--context-limit", type=int, default=None, help="codex: the project_doc_max_bytes your Codex config sets (default 32768, Codex's own); " @@ -97,6 +98,8 @@ def main(argv=None) -> int: from hermit.hosts.codex import prepare elif args.host == "pi": from hermit.hosts.pi import prepare + elif args.host == "mini-swe-agent": + from hermit.hosts.mini_swe_agent import prepare else: from hermit.hosts.claude_code import prepare try: diff --git a/hermit/config.py b/hermit/config.py index 70e65649..67b6155f 100644 --- a/hermit/config.py +++ b/hermit/config.py @@ -44,10 +44,10 @@ def _merge(base: dict, extra: dict, prefix: str = "") -> dict: FREE = ("params", "llm", "backends", "container", "resources", # sections whose keys the framework does not know - "model", *("codex", "claude", "pi", "react", "direct")) # a layer's settings (docs/framework/agent-layers.md) -BASES = {"codex": "codex", "claude": "claude-code", "claude-code": "claude-code", "pi": "pi", "react": "react", - "direct": None} -CLI_BASES = ("codex", "claude-code", "pi") # the bases that are a CLI (their Layer 1 settings: _model) + "model", *("codex", "claude", "pi", "mini-swe-agent", "react", "direct")) # a layer's (agent-layers.md) +BASES = {"codex": "codex", "claude": "claude-code", "claude-code": "claude-code", "pi": "pi", + "mini-swe-agent": "mini-swe-agent", "react": "react", "direct": None} +CLI_BASES = ("codex", "claude-code", "pi", "mini-swe-agent") # the bases that are a CLI (Layer 1 settings: _model) MODEL_KEYS = ("login", "effort", "price") # Layer 1 settings a CLI takes beside its model # A login's plan settings: Layer 2's (agent., evolver.), refused beside the model's name. PLAN_KEYS = {"plan_dir", "plan_max_wait_s", "plan_rerun", "usage_credits", "login_root"} @@ -114,15 +114,16 @@ def layered(data: dict) -> None: agent = data["agent"] base = agent.pop("base", None) model = agent.pop("model", None) - sections = {b: agent.pop(b, None) for b in ("codex", "claude", "pi", "react", "direct")} + sections = {b: agent.pop(b, None) for b in ("codex", "claude", "pi", "mini-swe-agent", "react", "direct")} if base is None: if model is not None or any(sections.values()): - raise ValueError("agent.model and agent. need agent.base (codex, claude, pi, react or direct)") + raise ValueError("agent.model and agent. need agent.base (codex, claude, pi, mini-swe-agent, react " + "or direct)") else: if base not in BASES: raise ValueError(f"agent.base must be one of {sorted(BASES)}, not {base!r}") builtin = {"codex": "codex", "claude-code": "claude-code", "claude": "claude-code", "pi": "pi", - "react": "react", "cli": None} + "mini-swe-agent": "mini-swe-agent", "react": "react", "cli": None} if not agent.get("name"): if BASES[base] is None: raise ValueError("agent.base: direct needs agent.name (the task's own loop calls it)") diff --git a/hermit/defaults.yaml b/hermit/defaults.yaml index 04e4243c..30bdf1bd 100644 --- a/hermit/defaults.yaml +++ b/hermit/defaults.yaml @@ -9,15 +9,18 @@ data: shuffle_seed: 0 task_info: null agent: - name: null # react | codex | claude-code | pi, or file.py:Class / file.py:function of the task's own + name: null # react | codex | claude-code | pi | mini-swe-agent, or file.py:Class / + # file.py:function of the task's own concurrency: 4 timeout_min: null # a sandboxed task call that runs longer is stopped and recorded as an error; null: none params: {} # Layer 3: the Acting Agent's own settings (docs/framework/agent-layers.md) - base: null # Layer 2: codex | claude | pi | react | direct; its settings go in a section of its name + base: null # Layer 2: codex | claude | pi | mini-swe-agent | react | direct; its settings go in a + # section of its name model: null # Layer 1: a / reference, or {ref | name, login, effort, price, ...} codex: null claude: null pi: null + mini-swe-agent: null react: null direct: null pipeline: # the older spelling of `agent`: run -> name diff --git a/hermit/hosts/mini_swe_agent.py b/hermit/hosts/mini_swe_agent.py new file mode 100644 index 00000000..e7f7385b --- /dev/null +++ b/hermit/hosts/mini_swe_agent.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Deploy an adapter into a mini-swe-agent workspace: the whole adapter under hermit//, and its exports +connected to mini-swe-agent's one place, the config file `mini-swe-agent.yaml`. + +mini-swe-agent (https://mini-swe-agent.com) reads no instructions file and has no plugins, sub-agents, skills, hooks +or MCP client: one YAML config's Jinja templates make its first two messages, and its model runs bash commands only. +So the config is Hermit's prompts over mini-swe-agent's own `mini` config (hermit.agent.base.mini_swe_agent), with +the adapter's context, the files it names and its tools written into the system template. The context names the +skills, roles, hooks and other files of the adapter's copy with what to do with them (hermit.hosts.inputs.resources), +as Pi's AGENTS.md does. The tools are one command, `hermit-tool ''`, answered in +mini-swe-agent's process by hermit.hosts.mini_swe_agent_env.AdapterEnvironment, which the config then names as its +environment class: mini-swe-agent must run in the Python environment hermit is installed in. An adapter without +tools keeps mini-swe-agent's `local` environment. + +Run it from the workspace with `mini -c mini-swe-agent.yaml -m `. The `mini-swe-agent` Base Agent reads the +same file's `hermit.context` and builds the same system message from Hermit's prompts, so an agent in training and +a deployed one read the same text (the Base Agent's own `model_class` and `config` are not deployed). Global +mini-swe-agent settings are never changed. +""" +import argparse +import json +from pathlib import Path +import sys + +import yaml + +from hermit.adapter.loader import agent_tools, hook_definitions, load_manifest, not_portable +from hermit.agent.base.mini_swe_agent import CONFIG_FILE, adapter_block, baked, render_config, tools_section +from hermit.hosts.inputs import (MARKER, adapter_files, adapter_folder, check_workspace, copy_adapter, folder_note, + render_exports, resources, source_label) + +HOST = "mini-swe-agent" +NATIVE: set[str] = set() # neither skills nor roles: the context names them +ENVIRONMENT_CLASS = "hermit.hosts.mini_swe_agent_env.AdapterEnvironment" + + +class _Dumper(yaml.SafeDumper): + pass + + +_Dumper.add_representer(str, lambda dumper, value: dumper.represent_scalar( + "tag:yaml.org,2002:str", value, style="|" if "\n" in value else None)) + + +def prepare(adapter: Path, output: Path, python: str, *, learning_material: Path | None = None, + context: str | None = None, serve_tools: bool = True) -> dict: + """`context`: the adapter's context for one task, which the harness computed, in place of the one rendered + here; an adapter whose context is per task deploys only so. `serve_tools`: false when the caller serves the + adapter's tools to the session itself (the `mini-swe-agent` Base Agent): the config then names no tools.""" + adapter, output = adapter.resolve(), output.resolve() + refused = not_portable(load_manifest(adapter), "A mini-swe-agent workspace") # before anything is written + if refused and context is None: + raise ValueError(refused) + previous = check_workspace(adapter, output, HOST) + manifest = load_manifest(adapter) + try: + config = render_config("mini") + except RuntimeError as error: # mini-swe-agent is not installed + raise ValueError(str(error)) from None + files = adapter_files(adapter, output, manifest, previous) + exports = render_exports(adapter, python) + listed, placed = resources(adapter, output, manifest, exports, hook_definitions(adapter, manifest), NATIVE) + text = "\n\n".join(t for t in (folder_note(output, manifest), + exports["context"] if context is None else context, listed) if t) + tools = agent_tools(manifest) if serve_tools else [] # the context tool is the harness's, not served + config["agent"]["system_template"] = baked(config["agent"]["system_template"], + adapter_block(text, tools_section(tools))) + audit = output / "tool-calls.jsonl" + if tools: + config["environment"] = {**config.get("environment", {}), "environment_class": ENVIRONMENT_CLASS, + "adapter": str(adapter), "audit_log": str(audit)} + if learning_material is not None: + config["environment"]["learning_material"] = str(Path(learning_material).resolve()) + config["hermit"] = {"context": text} # what the Base Agent reads (mini-swe-agent ignores the key) + output.mkdir(parents=True, exist_ok=True) + header = (f"# Generated by hermit for the adapter {manifest['id']} {manifest['version']} " + f"({source_label(manifest)}). No global settings are changed.\n" + "# Run from this directory: mini -c mini-swe-agent.yaml -m \n") + (output / CONFIG_FILE).write_text(header + yaml.dump(config, Dumper=_Dumper, sort_keys=False, allow_unicode=True, + width=10_000)) + copy_adapter(adapter, output, manifest, files, previous) + result = {"host": HOST, "adapter_id": manifest["id"], "version": manifest["version"], "adapter": str(adapter), + "workspace": str(output), "context_injected_via": CONFIG_FILE, + "environment_class": ENVIRONMENT_CLASS if tools else "local", "tools": [t["name"] for t in tools], + "roles": {}, "role_name_map": {}, "skills_dir": None, "skills": [], "hooks": None, + "audit_log": str(audit) if tools else None, "adapter_dir": str(adapter_folder(output, manifest)), + "files": files, "placed": placed, + "command": [python, "-m", "minisweagent", "-c", CONFIG_FILE, "-m", ""]} + (output / MARKER).write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--adapter", type=Path, required=True) + parser.add_argument("--workspace", "--output", dest="workspace", type=Path, required=True) + parser.add_argument("--python", default=sys.executable) + args = parser.parse_args() + print(json.dumps(prepare(args.adapter, args.workspace, str(Path(args.python).absolute())), ensure_ascii=False, + indent=2)) diff --git a/hermit/hosts/mini_swe_agent_env.py b/hermit/hosts/mini_swe_agent_env.py new file mode 100644 index 00000000..5639ee8a --- /dev/null +++ b/hermit/hosts/mini_swe_agent_env.py @@ -0,0 +1,50 @@ +"""AdapterEnvironment: a deployed mini-swe-agent workspace's environment, its adapter's tools as one bash command. + +`hermit deploy --host mini-swe-agent` names this class in the workspace's config when the adapter has tools; it is +hermit.agent.base.mini_swe_agent_env.HermitEnvironment with the tools answered in mini-swe-agent's process through a +Session of the deployed adapter (`adapter`), made at the first tool call, each call appended to `audit_log`. +Commands run as mini-swe-agent's LocalEnvironment runs them. This module imports mini-swe-agent: mini-swe-agent +imports it from the config, so `mini` must run in the Python environment Hermit is installed in. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +from hermit.agent.base.mini_swe_agent_env import HermitEnvironment, HermitEnvironmentConfig, text + +if TYPE_CHECKING: + from hermit.adapter.session import Session + + +class AdapterEnvironmentConfig(HermitEnvironmentConfig): + adapter: str = "" + """The deployed adapter whose tools `hermit-tool` runs.""" + audit_log: str = "" + """Every tool call is appended to this JSONL file (the workspace's tool-calls.jsonl).""" + learning_material: str = "" + """Bound to the adapter's tools as their learning material, when set.""" + + +class AdapterEnvironment(HermitEnvironment): + def __init__(self, *, config_class: type = AdapterEnvironmentConfig, **kwargs): + super().__init__(config_class=config_class, **kwargs) + self._adapter_session: Session | None = None + + def tool_list(self) -> list[dict]: + return self._session().tools if self.config.adapter else [] + + def call_tool(self, name: str, arguments: dict) -> str: + session = self._session() + if not session.has_tool(name): # the context tool included: it is the harness's, not the agent's + raise ValueError(f"Unknown tool: {name}") + return text(session.call(name, arguments)) + + def _session(self) -> Session: + if self._adapter_session is None: + from hermit.adapter.session import Session + self._adapter_session = Session(self.config.adapter, audit_log=self.config.audit_log or None, + learning_material=self.config.learning_material or None) + return self._adapter_session + + +__all__ = ["AdapterEnvironment", "AdapterEnvironmentConfig"] diff --git a/hermit/isolate.py b/hermit/isolate.py index 293d5c98..587bdabf 100644 --- a/hermit/isolate.py +++ b/hermit/isolate.py @@ -165,8 +165,8 @@ def validate(config) -> None: login = params.get("login") if kind and login: # wherever the pipeline runs: else every session would be refused, one task at a time own = codex_login.LOGINS.get(login) - if kind == "pi": - raise ValueError("agent pi takes no login: its model is a / reference (agent.model)") + if kind in ("pi", "mini-swe-agent"): + raise ValueError(f"agent {kind} takes no login: its model is a / reference (agent.model)") if own is None: raise ValueError(f"agent.params.login is {' or '.join(sorted(codex_login.LOGINS))} or absent, not " f"{login!r}") diff --git a/pyproject.toml b/pyproject.toml index 6d28ff07..6bba7b6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,8 @@ test = ["pytest"] # The bubblewrap sandboxes show the framework's environment: what is installed here is what agents and learners can # import (python -m pip install "hermit-harness[sandbox]"). sandbox = ["numpy>=1.26", "pandas>=2.2", "scipy>=1.13", "torch>=2.8"] +# The mini-swe-agent Base Agent and deploy host (hermit.agent.base.mini_swe_agent); it brings litellm. +mini-swe-agent = ["mini-swe-agent==2.4.6"] # Source-only method dependencies: groups are not included in PyPI package metadata. # From the checkout root, use pip >=25.1: python -m pip install -e . --group . @@ -64,7 +66,8 @@ Changelog = "https://github.com/Infini-AI-Lab/hermit/blob/main/CHANGELOG.md" include = ["hermit*"] [tool.setuptools.package-data] -hermit = ["defaults.yaml", "agent/model/backends.yaml", "agent/evolver/methods/**/*", "adapter/templates/**/*", +hermit = ["defaults.yaml", "agent/model/backends.yaml", "agent/base/mini_swe_agent.yaml", + "agent/evolver/methods/**/*", "adapter/templates/**/*", "agent/evolver/methods/**/.lint_blacklist", "adapter/templates/**/.agents/**/*", "adapter/templates/**/.codex/**/*", "adapter/templates/**/.hermit/**/*"] # globs skip dot-directories diff --git a/tests/core/test_context_tool.py b/tests/core/test_context_tool.py index 5cfcddbd..ba9a81b7 100644 --- a/tests/core/test_context_tool.py +++ b/tests/core/test_context_tool.py @@ -121,13 +121,13 @@ def test_the_codex_agent_gives_each_task_its_own_context(adapter, tmp_path): # ---------- plugin-portable: whether a template's adapters can be deployed or converted ---------- def test_a_template_that_is_not_plugin_portable_is_refused_by_deploy_and_convert(adapter, tmp_path): - from hermit.hosts import claude_code, codex, pi + from hermit.hosts import claude_code, codex, mini_swe_agent, pi from hermit.plugins import TARGETS, Unsupported, build for target in TARGETS: with pytest.raises(Unsupported, match="plugin-portable: false"): build(target, adapter, tmp_path / f"plugin-{target}", sys.executable) assert not (tmp_path / f"plugin-{target}").exists() - for host in (codex, claude_code, pi): + for host in (codex, claude_code, pi, mini_swe_agent): workspace = tmp_path / f"ws-{host.__name__.rsplit('.', 1)[-1]}" with pytest.raises(ValueError, match="plugin-portable: false"): host.prepare(adapter, workspace, sys.executable) diff --git a/tests/core/test_mini_swe_agent.py b/tests/core/test_mini_swe_agent.py new file mode 100644 index 00000000..97ab7716 --- /dev/null +++ b/tests/core/test_mini_swe_agent.py @@ -0,0 +1,418 @@ +"""The `mini-swe-agent` Base Agent (hermit.agent.base.cli.MiniSweAgent, its process +hermit.agent.base.mini_swe_agent_run), the `mini-swe-agent` acting agent built on it (hermit.agent.acting.workspace) +and the deploy host (hermit.hosts.mini_swe_agent). The stand-in is the model: an OpenAI-compatible chat/completions +server answering from scripted assistant messages, its payload in the shape OpenAI's API returns; mini-swe-agent's +own loop, environment and litellm run unchanged. Covered: the messages, the tools as the `hermit-tool` command, what +the commands' environment holds, the stream read into the reply, trajectory and usage, the limits, retries, what is +refused, and the deployed config giving `mini` the system message the Base Agent sends.""" +from http.server import BaseHTTPRequestHandler, HTTPServer +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import threading + +from jinja2 import StrictUndefined, Template +import pytest +import yaml + +from hermit.agent.acting.agent import load_agent +from hermit.agent.base import mini_swe_agent as mini +from hermit.agent.base.interface import Unsupported, make +from hermit.agent.base.trajectory import log_steps +from hermit.agent.model.backends import bind_agent_params, registry +from hermit.hosts import mini_swe_agent as host +from test_isolate import needs_bwrap # noqa: F401 (a marker, not a fixture: the bubblewrap check it carries) +from toy_task import make_task + +pytestmark = pytest.mark.skipif(importlib.util.find_spec("minisweagent") is None, + reason="needs mini-swe-agent: python -m pip install -e '.[mini-swe-agent]'") + +ROOT = Path(__file__).resolve().parents[2] +FILE_ADAPTER = ROOT / "docs" / "_examples" / "file_adapter" +KEY = "sk-test-not-a-real-key-7f3a" +PRICE = {"input": 1.0, "cached_input": 0.5, "output": 2.0} + + +class Scripted: + """An OpenAI chat/completions server answering from scripted assistant messages; it records each request's body + and Authorization header. A reply `(status, message)` is an error response.""" + + def __init__(self, replies): + self.replies, self.requests, self.auth = list(replies), [], [] + outer = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + outer.requests.append(json.loads(self.rfile.read(int(self.headers["Content-Length"])))) + outer.auth.append(self.headers.get("Authorization")) + message = outer.replies.pop(0) + if isinstance(message, tuple): + status, payload = message[0], {"error": {"message": message[1], "type": "invalid_request_error"}} + else: + status = 200 + payload = {"id": f"r{len(outer.requests)}", "object": "chat.completion", "created": 0, "model": "m", + "choices": [{"index": 0, "message": message, + "finish_reason": "tool_calls" if message.get("tool_calls") else "stop"}], + "usage": {"prompt_tokens": 100, "completion_tokens": 10, "total_tokens": 110, + "prompt_tokens_details": {"cached_tokens": 40}}} + data = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *args): + pass + self.server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.url = f"http://127.0.0.1:{self.server.server_port}/v1" + + def close(self): + self.server.shutdown() + self.server.server_close() + + +@pytest.fixture +def scripted(): + servers = [] + + def start(replies): + servers.append(Scripted(replies)) + return servers[-1] + yield start + for server in servers: + server.close() + + +def bash(command, cid): + return {"role": "assistant", "content": f"Running {cid}.", "tool_calls": [ + {"id": cid, "type": "function", "function": {"name": "bash", "arguments": json.dumps({"command": command})}}]} + + +def text(command): + return {"role": "assistant", "content": f"THOUGHT: next.\n\n```mswea_bash_command\n{command}\n```"} + + +def submit(cid="done"): + return bash("echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT && cat answer.txt", cid) + + +def backends(server): + return registry({"scripted": {"base_url": server.url, "api_key_env": "HERMIT_TEST_KEY", "litellm": "openai", + "apis": ["chat"], "prices": {"m": PRICE}}}, user_file=None) + + +def params(server, **more): + """The acting agent's params as a run config gives them: the model reference bound as normalize_config binds it.""" + return bind_agent_params({"model": "scripted/m", "web_search": False, "workspaces": "ws", **more}, + "mini-swe-agent", backends(server)) + + +def adapter_copy(tmp_path): + adapter = tmp_path / "adapter" + shutil.copytree(FILE_ADAPTER, adapter) + return adapter + + +def test_the_acting_agent_runs_the_adapters_tools_without_the_key_and_submits(tmp_path, monkeypatch, scripted): + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + monkeypatch.setenv("UNRELATED_SECRET", "do-not-leak") + monkeypatch.setenv("MY_TOKEN", "tok-expanded-value") + server = scripted([ + bash("hermit-tool count_words '{\"text\": \"one two three\"}'", "c1"), + bash("hermit-tool nope '{}'", "c2"), + bash("echo key=${HERMIT_TEST_KEY:-unset} secret=${UNRELATED_SECRET:-unset} token=$TOKEN", "c3"), + bash("head -3 hermit/file-example/skills/verify-output/SKILL.md", "c4"), + bash("printf '42\\n' > answer.txt", "c5"), + submit(), + ]) + agent = load_agent("mini-swe-agent", params(server, prompt="Q: {question}", env={"TOKEN": "$MY_TOKEN"}), + tmp_path) + out = agent.run({"id": "t1", "question": "What is six times seven?"}, adapter_copy(tmp_path)) + + assert out.output == "42" and out.extra["exit_code"] == 0 and out.extra["backend"] == "mini-swe-agent" + system, user = server.requests[0]["messages"][:2] + assert "Review the task requirements before acting." in system["content"] # ADAPTER.md + assert "`hermit/file-example/skills/verify-output/SKILL.md`" in system["content"] # its skills and roles, named + assert "`reviewer`" in system["content"] and "`count_words`" in system["content"] + assert "Please solve this task: Q: What is six times seven?" in user["content"] + assert [t["function"]["name"] for t in server.requests[0]["tools"]] == ["bash"] + assert server.auth == [f"Bearer {KEY}"] * 6 # the key goes with each request + steps = log_steps(out.log) + commands = [s for s in steps if s.kind == "command"] + assert [s.exit_code for s in commands] == [0, 1, 0, 0, 0, 0] + assert json.loads(commands[0].output) == {"count": 3} and "Unknown tool: nope" in commands[1].output + assert "key=unset secret=unset token=tok-expanded-value" in commands[2].output + assert "name: verify-output" in commands[3].output and commands[5].output == "42\n" + # Six responses of 100 prompt tokens (40 cached) and 10 output tokens, at the backend's price. + assert out.tokens == 660 and out.api_speed == {"model_calls": 6} + assert out.cost_usd == pytest.approx((6 * 60 * 1.0 + 6 * 40 * 0.5 + 6 * 10 * 2.0) / 1e6) + run_dir = Path(out.log).parent + assert (run_dir / "work" / "answer.txt").read_text() == "42\n" + for path in run_dir.rglob("*"): # the stream, the deployment, the workspace: no key, no token + if path.is_file(): + assert KEY not in path.read_text(errors="replace"), path + if path.name != "agent-mini-swe-agent.jsonl": # the stream holds the command's output, as it ran + assert "tok-expanded-value" not in path.read_text(errors="replace"), path + + +def test_the_text_based_model_class_answers_in_bash_blocks_with_no_adapter(tmp_path, monkeypatch, scripted): + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + server = scripted([text("echo hello > answer.txt"), + text("echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT && cat answer.txt")]) + out = load_agent("mini-swe-agent", params(server, model_class="litellm_textbased"), tmp_path).run( + {"id": "t2", "prompt": "Say hello."}, None) + assert out.output == "hello" and "tools" not in server.requests[0] + assert "mswea_bash_command" in server.requests[0]["messages"][0]["content"] + assert mini.CONTEXT_HEADING not in server.requests[0]["messages"][0]["content"] + + +def test_the_step_limit_ends_a_task_with_its_last_message_as_the_answer(tmp_path, monkeypatch, scripted): + """The config file's agent.step_limit wins over the setting, and its model section is merged (cost_tracking + there does not collide with Hermit's own).""" + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + server = scripted([bash("echo one", "c1"), bash("echo two", "c2"), bash("echo three", "c3")]) + (tmp_path / "mini.yaml").write_text("agent:\n step_limit: 2\nmodel:\n cost_tracking: default\n") + out = load_agent("mini-swe-agent", params(server, config="mini.yaml", step_limit=9), tmp_path).run( + {"id": "t3", "prompt": "Count."}, None) + assert len(server.requests) == 2 and out.output == "Running c2." and out.extra["exit_code"] == 0 + assert [s.text for s in log_steps(out.log) if s.kind == "error"] == ["LimitsExceeded: LimitsExceeded"] + + +def test_a_failed_model_call_fails_the_session_with_the_calls_it_made(tmp_path, monkeypatch, scripted): + """A context overflow breaks the session off: it failed, with no answer to judge (an error record, as a CLI's + failed session is), and the tokens it spent still count.""" + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + server = scripted([bash("echo hi", "c1"), (400, "This model's maximum context length is 8192 tokens.")]) + out = load_agent("mini-swe-agent", params(server, answer="(.*?)"), tmp_path).run( + {"id": "t4", "prompt": "Go."}, None) + assert out.extra["exit_code"] == 1 and out.output is None + assert "ContextWindowExceededError" in out.extra["session_failed"] + assert out.tokens == 110 and out.api_speed == {"model_calls": 1} + assert "ContextWindowExceededError" in [s for s in log_steps(out.log) if s.kind == "error"][-1].text + + +def test_a_process_cut_off_after_the_model_answered_fails_the_session(tmp_path, monkeypatch, scripted): + """A session ends with mini-swe-agent's exit message; the process killed before it (here by its own command, as + an out-of-memory kill would) leaves none, and the session failed: not an empty answer to judge.""" + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + server = scripted([bash("kill -9 $PPID", "c1")]) + out = load_agent("mini-swe-agent", params(server), tmp_path).run({"id": "t12", "prompt": "Go."}, None) + assert out.output is None and "session_failed" in out.extra and len(server.requests) == 1 + + +def test_the_settings_mini_swe_agent_cannot_take_are_refused_before_any_task(tmp_path): + from hermit.config import Config + for more, match in ((dict(login="chatgpt"), "takes no login"), (dict(isolation="container"), "isolation")): + with pytest.raises(ValueError, match=match): + load_agent("mini-swe-agent", {"model": "m", "web_search": False, **more}, tmp_path).run( + {"id": "t", "prompt": "x"}, None) + path = make_task(tmp_path, sandbox={"enabled": True}, model="deepseek/deepseek-flash", + agent={"name": "mini-swe-agent", "params": {"web_search": False, "login": "chatgpt"}}) + with pytest.raises(ValueError, match="agent mini-swe-agent takes no login"): + Config.load(path, []) + + +def test_a_provider_that_stops_serving_midway_is_tried_again_not_scored(tmp_path, monkeypatch, scripted): + """A rate limit that mini-swe-agent's own retries could not get past, after the model has answered once, is the + provider's failure, not the task's: the task runs again in a fresh workspace (CliAgent's UNAVAILABLE, through the + event's `unavailable`), and the first attempt's mid-task text is never the answer.""" + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + monkeypatch.setenv("MSWEA_MODEL_RETRY_STOP_AFTER_ATTEMPT", "1") # mini-swe-agent's own: once, no wait + (tmp_path / "mini.yaml").write_text("model:\n model_kwargs:\n max_retries: 0\n") # nor the client's + server = scripted([bash("echo looking", "c0"), (429, "Rate limit reached for requests."), + bash("printf 'ok' > answer.txt", "c1"), submit()]) + out = load_agent("mini-swe-agent", params(server, config="mini.yaml", retries=2, retry_wait_s=0), tmp_path).run( + {"id": "t7", "prompt": "Go."}, None) + assert out.output == "ok" and out.extra["attempts"] == 2 and len(server.requests) == 4 + + +@pytest.mark.parametrize("status,error", [(401, "Incorrect API key provided."), (404, "The model m does not exist."), + (400, "Authentication Fails, Your api key is invalid")]) # DeepSeek's +def test_a_wrong_key_or_model_on_the_first_call_stops_the_agent_rather_than_score(tmp_path, monkeypatch, scripted, + status, error): + """A setting every task shares, found wrong by the first model call, is no task's wrong answer: no session starts + (the events are held back until the model answers) and the acting agent raises.""" + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + server = scripted([(status, error)]) + with pytest.raises(RuntimeError, match="before starting a session"): + load_agent("mini-swe-agent", params(server), tmp_path).run({"id": "t10", "prompt": "Go."}, None) + assert len(server.requests) == 1 + + +def test_a_timeout_stops_the_command_that_is_running(tmp_path, monkeypatch, scripted): + """A command runs in a session of its own, which the Base Agent's kill of the process group does not reach (on + macOS nothing else finds it): the process kills it when it is stopped. The delay is what the test holds.""" + import time + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + marker = tmp_path / "still-running" + server = scripted([bash(f"sleep 6; touch {marker}", "c1")]) + started = time.time() + out = load_agent("mini-swe-agent", params(server, timeout_s=5), tmp_path).run({"id": "t11", "prompt": "Go."}, None) + assert out.extra["timed_out"] is True and len(server.requests) == 1 + time.sleep(max(0.0, started + 9 - time.time())) # past when the command would have finished + assert not marker.exists() + + +def test_a_module_in_the_workspace_is_not_imported_by_minis_process(tmp_path, monkeypatch, scripted): + """The process runs in the workspace; a task's file named like a module it imports (a repository with its own + `yaml.py`) must not replace that module in the process that holds the model's key.""" + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + (tmp_path / "yaml.py").write_text("raise SystemExit(7)\n") + server = scripted([bash("printf 'ok' > answer.txt", "c1"), submit()]) + out = load_agent("mini-swe-agent", params(server, files=["yaml.py"]), tmp_path).run({"id": "t8", "prompt": "Go."}, + None) + assert out.output == "ok" and (Path(out.extra["workspace"]) / "yaml.py").is_file() + + +@pytest.mark.skipif(not sys.platform.startswith("linux") or os.geteuid() == 0, + reason="/proc and PR_SET_DUMPABLE are Linux's; root reads any process's environment") +def test_a_command_cannot_read_the_key_from_minis_process(tmp_path, monkeypatch, scripted): + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + server = scripted([bash("tr '\\0' '\\n' < /proc/$PPID/environ | grep -c HERMIT_TEST_KEY; true", "c1"), + bash("printf 'ok' > answer.txt", "c2"), submit()]) + out = load_agent("mini-swe-agent", params(server), tmp_path).run({"id": "t9", "prompt": "Go."}, None) + first = [s for s in log_steps(out.log) if s.kind == "command"][0] + assert out.output == "ok" and "Permission denied" in first.output and KEY not in first.output + + +def test_what_mini_swe_agent_cannot_do_is_refused(tmp_path, monkeypatch, scripted): + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + agent = make("mini-swe-agent", {"web_search": False, "model": "openai/m"}) + for call, match in ((dict(session="s1"), "resume"), (dict(roles=[{"name": "r"}]), "sub-agents")): + with pytest.raises(Unsupported, match=match): + agent.run("x", workspace=tmp_path, **call) + for setting, match in ((dict(web_search=True), "web search"), (dict(login="chatgpt"), "no login"), + (dict(isolation="container"), "isolation")): + with pytest.raises(Unsupported, match=match): + agent.configured(**setting).run("x", workspace=tmp_path) + with pytest.raises(ValueError, match="needs a model"): + make("mini-swe-agent", {"web_search": False}).run("x", workspace=tmp_path) + # another model class calls its own service with what it is given: never a binding's key and URL + bound = make("mini-swe-agent", {"web_search": False, "binding": {"base_url": "http://x", "model": "m"}, + "model_class": "openrouter"}) + with pytest.raises(Unsupported, match="calls its own service"): + bound.run("x", workspace=tmp_path) + # a config that names the model or the environment class fails every task: no session starts, and the acting + # agent reports the error rather than score an empty answer + server = scripted([]) + (tmp_path / "bad.yaml").write_text("model:\n model_name: other\nenvironment:\n environment_class: docker\n") + with pytest.raises(RuntimeError, match="model.model_name and environment.environment_class"): + load_agent("mini-swe-agent", params(server, config="bad.yaml"), tmp_path).run({"id": "t5", "prompt": "x"}, None) + assert server.requests == [] + + +def system_message(config: dict, workspace: Path) -> str: + """The system message `mini -c` renders from a deployed config: its template with mini-swe-agent's variables (the + environment's include this process's variables).""" + import platform + return Template(config["agent"]["system_template"], undefined=StrictUndefined).render( + **{**config["agent"], **platform.uname()._asdict(), **os.environ, "cwd": str(workspace)}) + + +@pytest.mark.parametrize("endraw", ["{% endraw %}", "{%- endraw -%}", "{%+ endraw %}", "{%endraw+%}", "{% endraw %}"]) +def test_a_deployed_config_renders_the_base_agents_system_message_whatever_the_context_holds( + tmp_path, monkeypatch, scripted, endraw): + """The adapter's context is literal text in the deployed template: braces, a raw block and every spelling of its + end stay text, so the learner's ADAPTER.md cannot become template code (a variable there could print a key).""" + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + adapter = adapter_copy(tmp_path) + (adapter / "ADAPTER.md").write_text("Use {{ HERMIT_TEST_KEY }} as {% raw %}{{ x }}" + endraw + " text.\n") + server = scripted([bash("printf 'ok' > answer.txt", "c1"), submit()]) + agent = load_agent("mini-swe-agent", params(server), tmp_path) + out = agent.run({"id": "t6", "prompt": "Go."}, adapter) + sent = server.requests[0]["messages"][0]["content"] + assert "Use {{ HERMIT_TEST_KEY }} as {% raw %}{{ x }}" + endraw + " text." in sent and KEY not in sent + + workspace = tmp_path / "deployed" + deployed = host.prepare(adapter, workspace, sys.executable) + config = yaml.safe_load((workspace / mini.CONFIG_FILE).read_text()) + work = Path(out.extra["workspace"]) + rendered = system_message(config, workspace) + assert rendered == sent.replace(str(work), str(workspace)) # the same text, the deployed workspace's paths + assert deployed["environment_class"] == host.ENVIRONMENT_CLASS and deployed["tools"] == ["count_words"] + + +def test_a_deployed_workspace_runs_the_adapters_tools_in_minis_own_environment(tmp_path): + adapter = adapter_copy(tmp_path) + workspace = tmp_path / "ws" + host.prepare(adapter, workspace, sys.executable) + config = yaml.safe_load((workspace / mini.CONFIG_FILE).read_text()) + mini.load_minisweagent() + from minisweagent.environments import get_environment + env = get_environment({**config["environment"], "cwd": str(workspace)}) + ran = env.execute({"command": "hermit-tool count_words '{\"text\": \"a b\"}'"}) + assert ran["returncode"] == 0 and json.loads(ran["output"]) == {"count": 2} + assert env.execute({"command": "hermit-tool count_words '{}' && ls"})["returncode"] == 1 # one command only + assert env.execute({"command": "echo plain"})["output"] == "plain\n" + calls = [json.loads(line) for line in (workspace / "tool-calls.jsonl").read_text().splitlines()] + assert [c["tool"] for c in calls] == ["count_words"] + + +def test_deploy_names_the_roles_keeps_the_local_environment_without_tools_and_redeploys(tmp_path): + adapter = adapter_copy(tmp_path) + exports = yaml.safe_load((adapter / "exports.yaml").read_text()) + del exports["tools"] + (adapter / "exports.yaml").write_text(yaml.safe_dump(exports, sort_keys=False)) + workspace = tmp_path / "ws" + result = host.prepare(adapter, workspace, sys.executable) + config = yaml.safe_load((workspace / mini.CONFIG_FILE).read_text()) + assert result["environment_class"] == "local" and "environment_class" not in config["environment"] + assert "`reviewer`" in config["hermit"]["context"] and result["placed"]["agents"] == "files" + assert (workspace / "hermit" / "file-example" / "agents" / "reviewer.md").is_file() + (adapter / "ADAPTER.md").write_text("Changed.\n") + host.prepare(adapter, workspace, sys.executable) # a redeploy replaces it + assert "Changed." in yaml.safe_load((workspace / mini.CONFIG_FILE).read_text())["hermit"]["context"] + with pytest.raises(ValueError, match="another host"): + from hermit.hosts import codex + codex.prepare(adapter, workspace, sys.executable) + run = subprocess.run([sys.executable, "-m", "hermit", "deploy", "--adapter", str(adapter), "--workspace", + str(tmp_path / "cli"), "--host", "mini-swe-agent"], capture_output=True, text=True, cwd=ROOT) + assert run.returncode == 0 and json.loads(run.stdout)["host"] == "mini-swe-agent" + + +def test_the_prompts_come_from_minis_package_not_a_mini_yaml_where_it_runs(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "mini.yaml").write_text("agent:\n system_template: hijacked\n") + monkeypatch.setenv("MSWEA_CONFIG_DIR", str(tmp_path)) + config = mini.render_config("mini") + assert config["agent"]["system_template"].startswith("You are a helpful assistant") and config["model"] + assert mini.base_config("litellm_textbased") == "mini_textbased" and mini.base_config("litellm") == "mini" + + +def test_the_layered_config_names_mini_swe_agent_and_binds_its_model(tmp_path): + from hermit.config import Config + path = make_task(tmp_path, agent={"base": "mini-swe-agent", "model": "deepseek/deepseek-flash", + "mini-swe-agent": {"web_search": False, "step_limit": 7}}) + agent = Config.load(path, ["agent.mini-swe-agent.model_class=litellm_textbased"]).data["agent"] + assert agent["name"] == "mini-swe-agent" + assert agent["params"]["binding"]["litellm"] == "deepseek/deepseek-flash" and agent["params"]["step_limit"] == 7 + assert agent["params"]["model_class"] == "litellm_textbased" and agent["params"]["model"] == "deepseek-flash" + + +@needs_bwrap +def test_a_rollout_in_the_task_sandbox_reaches_its_model_but_not_the_runs_data(tmp_path, monkeypatch, scripted): + """In the task sandbox the Base Agent's process reaches its model on the loopback (forwarded), and a command it + runs does not see the run's data.""" + from hermit.config import Config + from hermit.isolate import SandboxedAgent + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + data = tmp_path / "tasks.jsonl" + server = scripted([bash(f"cat {data} 2>&1 | head -1; printf '9' > answer.txt", "c1"), submit()]) + path = make_task(tmp_path, sandbox={"enabled": True}, model="scripted/m", + backends={"scripted": {"base_url": server.url, "api_key_env": "HERMIT_TEST_KEY", + "litellm": "openai", "apis": ["chat"]}}, + agent={"name": "mini-swe-agent", "concurrency": 1, "params": {"web_search": False}}) + agent = SandboxedAgent(Config.load(path, []), "rollout", records=None) + try: + out = agent({"id": "t1", "prompt": "Write 9."}, None) + finally: + agent.close() + assert out.output == "9" and len(server.requests) == 2 + first = [s for s in log_steps(Path(out.log)) if s.kind == "command"][0] + assert "No such file" in first.output and data.is_file() # there outside, not shown inside From 844897b9c6273de59ab839dd26df0343f97bf994 Mon Sep 17 00:00:00 2001 From: Haizhong Zheng Date: Mon, 9 Nov 2026 17:57:09 -0500 Subject: [PATCH 2/2] mini-swe-agent: review fixes (empty submission, model classes, its own sandbox, core-only test collection) - after a format error mini's own message told the model to end with the bare COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT, an empty submission: Hermit's format_error_template ends with `... && cat answer.txt` - model_class is litellm or litellm_textbased in every case: the *_response classes' messages and the openrouter / portkey classes' errors are not litellm's (usage uncounted, a rate limit not retried) - under isolation: sandbox the home names no host path, as Pi's: the config is read into the settings, the system instructions go on the command (past ARG_MAX in a file, Claude Code's _prompt_file, now CliAgent's) - a thinking model's reasoning_content is a reasoning step of the trajectory - tests/core collects without mini-swe-agent (jinja2 imported only when present) - docs: the tutorial's time-limit sentence; agent.base / agent. rows in config-fields Co-Authored-By: Claude Opus 5.5 --- docs/reference/config-fields.md | 4 +- docs/tutorials/mini-swe-agent.md | 3 +- hermit/agent/base/cli.py | 41 ++++++++------ hermit/agent/base/mini_swe_agent.py | 6 +- hermit/agent/base/mini_swe_agent.yaml | 25 ++++++++- hermit/agent/base/mini_swe_agent_run.py | 12 ++-- hermit/agent/base/trajectory.py | 2 + tests/core/test_mini_swe_agent.py | 75 ++++++++++++++++++++++++- 8 files changed, 142 insertions(+), 26 deletions(-) diff --git a/docs/reference/config-fields.md b/docs/reference/config-fields.md index 19f84ced..f2851230 100644 --- a/docs/reference/config-fields.md +++ b/docs/reference/config-fields.md @@ -36,9 +36,9 @@ guarantee that their consumer supports arbitrary fields. | `agent.concurrency` | integer / 4 | Maximum concurrent rollout/evaluation task executions; use a positive value | | `agent.timeout_min` | number / null | A sandboxed task call (its worker) running longer is stopped and recorded as an error; null: no limit | | `agent.params` | mapping / {} | Passed to Agent constructors; plain functions receive it when they declare a `params` argument | -| `agent.base` | string / null | Layer 2 in the per-layer form ([Agent layers](../framework/agent-layers.md#settings-by-layer)): `codex`, `claude`, `pi`, `react`, or `direct` (with `agent.name`, a task's own loop); sets `agent.name` when unset | +| `agent.base` | string / null | Layer 2 in the per-layer form ([Agent layers](../framework/agent-layers.md#settings-by-layer)): `codex`, `claude`, `pi`, `mini-swe-agent`, `react`, or `direct` (with `agent.name`, a task's own loop); sets `agent.name` when unset | | `agent.model` | string / mapping / null | Layer 1: a `/` reference, or `{ref` or `name, login, effort, price, ...}`; read into `agent.params` | -| `agent.codex`, `agent.claude`, `agent.pi`, `agent.react`, `agent.direct` | mapping / null | Layer 2's settings for that Base Agent (the one `agent.base` names); read into `agent.params`. A key set in two layers is refused | +| `agent.codex`, `agent.claude`, `agent.pi`, `agent.mini-swe-agent`, `agent.react`, `agent.direct` | mapping / null | Layer 2's settings for that Base Agent (the one `agent.base` names); read into `agent.params`. A key set in two layers is refused | | `pipeline.run` | string / null | Legacy alias for `agent.name`, applied only when `agent.name` is unset | | `pipeline.concurrency` | integer / null | Legacy concurrency override when the legacy pipeline alias is used | | `pipeline.params` | mapping / {} | Legacy parameters merged into `agent.params` when the pipeline alias is used | diff --git a/docs/tutorials/mini-swe-agent.md b/docs/tutorials/mini-swe-agent.md index c0877684..d91a6872 100644 --- a/docs/tutorials/mini-swe-agent.md +++ b/docs/tutorials/mini-swe-agent.md @@ -56,7 +56,8 @@ settings in `agent.mini-swe-agent`. The task functions of the [CLI agents](../fr Each task runs in a fresh workspace (`work/`, with the log in `log/` beside it). The agent submits its answer as mini-swe-agent ends a task, `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT && cat answer.txt`, and the submission is the -last message the judge scores; without one (the step limit, the time limit) it is the last assistant message. The +last message the judge scores; without one (the step limit) it is the last assistant message, and a task stopped at +the time limit has none. The log holds the session's events (`agent-mini-swe-agent.jsonl`), which every Evolver reads as steps: the messages, each command with its output and exit code, errors. diff --git a/hermit/agent/base/cli.py b/hermit/agent/base/cli.py index 1c1c5a3e..ec605115 100644 --- a/hermit/agent/base/cli.py +++ b/hermit/agent/base/cli.py @@ -494,6 +494,16 @@ def home_env(self, home: Path) -> dict[str, str]: """The variables that point the CLI at the session's home.""" return {self.home_var: str(home)} + def _prompt_file(self, flag: str, text: str, home: Path) -> list[str]: + """A system text for the command line, or past what one argument may hold in a file of the session's home, the + one place every isolation shows it.""" + if len(text.encode()) <= ARG_MAX: + return [flag, text] + path = home / f"{flag.lstrip('-')}.md" # system-prompt.md, append-system-prompt.md: one file each + _clear(path) # a link the last session left would take the text elsewhere + path.write_text(text, encoding="utf-8") + return [f"{flag}-file", str(path)] + def prepare_home(self, home: Path, roles: Sequence[dict], tools_url: str | None, tools_name: str) -> None: """The session's configuration in its home, before the call (nothing, for a CLI given it on its command line).""" @@ -1152,16 +1162,6 @@ def by_fd(self): commands' holds it (and the image's CLI is undumpable: nothing beside it reads it from its memory).""" return {codex_login.TOKEN_ENV: CLAUDE_TOKEN_FD} if self.settings.get("login") == "claude" else {} - def _prompt_file(self, flag: str, text: str, home: Path) -> list[str]: - """A system text for the command line, or past what one argument may hold in a file of the session's home, the - one place every isolation shows it.""" - if len(text.encode()) <= ARG_MAX: - return [flag, text] - path = home / f"{flag.lstrip('-')}.md" # system-prompt.md, append-system-prompt.md: one file each - _clear(path) # a link the last session left would take the text elsewhere - path.write_text(text, encoding="utf-8") - return [f"{flag}-file", str(path)] - def command(self, prompt, workspace, last_file, *, session=None, roles=(), tools_url=None): s = self.settings text = bool(s.get("text_only")) @@ -1476,7 +1476,7 @@ class MiniSweAgent(CliAgent): binding the resolved `/` reference (hermit.agent.model.backends): its URL, key variable, extra body and price model_class litellm (its bash tool, the default) or litellm_textbased (one ```mswea_bash_command block per - response); without a binding, any mini-swe-agent model class name + response); another provider's model goes through litellm (`openrouter/`) effort sent as litellm's reasoning_effort config a mini-swe-agent config file merged over Hermit's (agent.step_limit, the templates, model.model_kwargs, environment.timeout, ...); not its model name or class, nor its environment @@ -1530,16 +1530,24 @@ def run(self, prompt, *, workspace=None, tools: Sequence[Tool] = (), limits: Lim # another class sends what it is given to its own service: the binding's key and URL among it raise Unsupported(f"mini-swe-agent's model_class {model_class!r} calls its own service; a " f"/ reference takes {' or '.join(self.binding_classes)}") + if model_class not in self.binding_classes: + # the others' messages (the *_response classes': the Responses API's) and errors (openrouter's, portkey's) + # are not litellm's: their usage, steps and a provider that could not serve would go unread + raise Unsupported(f"mini-swe-agent's model_class {model_class!r}: Hermit reads " + f"{' and '.join(self.binding_classes)} only; another provider's model goes through " + "litellm (openrouter/, ...)") return super().run(prompt, workspace=workspace, tools=tools, limits=limits, session=session, roles=roles) def prepare_home(self, home, roles, tools_url, tools_name): - """The task's settings for the runner, in its home: no key (the runner reads the key's variable), and of - `env` only the names (the runner passes those variables to the commands).""" + """The task's settings for the runner, in its home: no key (the runner reads the key's variable), of `env` + only the names (the runner passes those variables to the commands), and no host path, as Pi's: a sandbox + translates only the command, the environment and the prompt, so the `config` file is read here and the + system instructions go on the command (a long one in a file, as Claude Code's).""" + import yaml s = self.settings - run = {k: s[k] for k in ("model", "binding", "model_class", "effort", "step_limit", "system") - if s.get(k) is not None} + run = {k: s[k] for k in ("model", "binding", "model_class", "effort", "step_limit") if s.get(k) is not None} if s.get("config"): - run["config"] = str(Path(os.path.expanduser(str(s["config"]))).resolve()) + run["config"] = yaml.safe_load(Path(os.path.expanduser(str(s["config"]))).read_text()) or {} run["env_names"] = sorted((s.get("env") or {})) path = home / "mini-swe-agent.json" if os.path.lexists(path) and not _regular(path): @@ -1552,6 +1560,7 @@ def command(self, prompt, workspace, last_file, *, session=None, roles=(), tools return [str(s.get("bin") or self.program), "-P", "-m", "hermit.agent.base.mini_swe_agent_run", "--settings", str(last_file.with_name("mini-swe-agent.json")), "--cwd", str(s.get("cwd") or workspace), "--last", str(last_file), *(["--stdin"] if s.get("stdin") else ["--prompt", prompt]), + *(self._prompt_file("--system", str(s["system"]), last_file.parent) if s.get("system") else []), *(s.get("args") or [])] @classmethod diff --git a/hermit/agent/base/mini_swe_agent.py b/hermit/agent/base/mini_swe_agent.py index 847ed402..a415fb48 100644 --- a/hermit/agent/base/mini_swe_agent.py +++ b/hermit/agent/base/mini_swe_agent.py @@ -46,6 +46,7 @@ _PLACEHOLDER = re.compile(r"\{\{-?\s*adapter_context\s*-?\}\}") _ENDRAW = re.compile(r"\{%[-+]?\s*endraw\s*[-+]?%\}") # every spelling Jinja ends a raw block with STEP_LIMIT = 50 +MODEL_PROMPTS = ("format_error_template",) # Hermit prompts that go in the model section INSTALL = ("mini-swe-agent is not installed: python -m pip install 'hermit-harness[mini-swe-agent]' " "(from a checkout: python -m pip install -e '.[mini-swe-agent]')") _LOADING = threading.Lock() @@ -90,7 +91,10 @@ def render_config(base: str = "mini", *overrides: dict) -> dict: if base not in prompts: raise ValueError(f"no Hermit prompts for mini-swe-agent's {base!r} config; known: {sorted(prompts)}") shipped = Path(list(spec.submodule_search_locations)[0]) / "config" / f"{base}.yaml" - return _merge(yaml.safe_load(shipped.read_text()), {"agent": prompts[base]}, *(o for o in overrides if o)) + agent = dict(prompts[base]) + model = {k: agent.pop(k) for k in MODEL_PROMPTS if k in agent} # the model section's, as mini-swe-agent has them + return _merge(yaml.safe_load(shipped.read_text()), {"agent": agent}, {"model": model}, + *(o for o in overrides if o)) def _merge(*configs: dict) -> dict: diff --git a/hermit/agent/base/mini_swe_agent.yaml b/hermit/agent/base/mini_swe_agent.yaml index 683439fa..47d1831c 100644 --- a/hermit/agent/base/mini_swe_agent.yaml +++ b/hermit/agent/base/mini_swe_agent.yaml @@ -3,7 +3,8 @@ # ```mswea_bash_command block per response). Each is that config's agent section from mini-swe-agent 2.4.6 with # two changes: the task is any task rather than an issue in a repository, and the agent submits its answer (the # output the judge scores) as mini-swe-agent's swebench config submits a patch. `{{ adapter_context }}` is where the -# adapter's context goes; the model and environment sections stay mini-swe-agent's own (hermit/agent/base/mini_swe_agent.py). +# adapter's context goes; the model and environment sections stay mini-swe-agent's own (hermit/agent/base/mini_swe_agent.py), +# but for `mini`'s format_error_template, whose way to end a task submits the answer too. mini: system_template: | You are a helpful assistant that can interact with a computer. @@ -104,6 +105,28 @@ mini: ```bash anything ``` + # the model section's format_error_template, with mini's ending changed to submit the answer as above + format_error_template: | + {% if finish_reason is defined and (finish_reason == "length" or (finish_reason == "tool_calls" and not has_tool_calls)) -%} + Your previous response reached the output token limit (finish_reason={{ finish_reason }}) before you produced a tool call, so it was cut off. Respond more concisely and finish with exactly one bash tool call. If you need to think more, do so briefly. + {%- else -%} + Tool call error: + + + {{error}} + + + Here is general guidance on how to submit correct toolcalls: + + Every response needs to use the 'bash' tool at least once to execute commands. + + Call the bash tool with your command as the argument: + - Tool: bash + - Arguments: {"command": "your_command_here"} + + If you want to end the task, please write your final answer to `answer.txt` and issue the following command: + `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT && cat answer.txt` without any other command. + {%- endif %} mini_textbased: system_template: | diff --git a/hermit/agent/base/mini_swe_agent_run.py b/hermit/agent/base/mini_swe_agent_run.py index 2f824d91..bcc70636 100644 --- a/hermit/agent/base/mini_swe_agent_run.py +++ b/hermit/agent/base/mini_swe_agent_run.py @@ -1,7 +1,7 @@ """One mini-swe-agent task, run as the `mini-swe-agent` Base Agent's process (hermit.agent.base.mini_swe_agent). python -m hermit.agent.base.mini_swe_agent_run --settings /mini-swe-agent.json --cwd \ - --last /last_message.txt --stdin < prompt + --last /last_message.txt [--system | --system-file ] --stdin < prompt The task runs as mini-swe-agent's batch runner runs one: a model, an environment and a DefaultAgent from its Python API, the config Hermit's prompts over mini-swe-agent's own (hermit.agent.base.mini_swe_agent.render_config) and the @@ -56,6 +56,8 @@ def event_of(message: dict) -> dict: for key in ("tool_calls", "tool_call_id"): if message.get(key) is not None: out[key] = message[key] + if message.get("reasoning_content"): # a thinking model's (litellm's field) + out["reasoning"] = str(message["reasoning_content"]) if extra.get("actions"): out["actions"] = [str(a.get("command", "")) for a in extra["actions"] if isinstance(a, dict)] if "returncode" in extra: @@ -112,8 +114,7 @@ def build(settings: dict, cwd: str, emit: Callable[[dict], None]): litellm.suppress_debug_info = True # its help text would go to stdout, between the events model_class = str(settings.get("model_class") or "litellm") - user = yaml.safe_load(Path(settings["config"]).read_text()) if settings.get("config") else None - user = user or {} + user = settings.get("config") or {} # the config file's settings, read by the Base Agent given = [f"model.{k}" for k in ("model_name", "model_class") if k in (user.get("model") or {})] given += ["environment.environment_class"] if "environment_class" in (user.get("environment") or {}) else [] if given: @@ -241,11 +242,14 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--last", type=Path, required=True) parser.add_argument("--prompt") parser.add_argument("--stdin", action="store_true") + parser.add_argument("--system", default="") # the call's system instructions (a sandbox translates its paths) + parser.add_argument("--system-file", type=Path) # or, past what one argument holds, the file they are in args = parser.parse_args(argv) _not_dumpable() signal.signal(signal.SIGTERM, _stop) prompt = sys.stdin.read() if args.stdin else (args.prompt or "") - settings = json.loads(args.settings.read_text()) + system = args.system_file.read_text(encoding="utf-8") if args.system_file else args.system + settings = {**json.loads(args.settings.read_text()), "system": system} args.last.write_text("", encoding="utf-8") stream = Stream({"type": "session.started", "session": uuid.uuid4().hex, "model": (settings.get("binding") or {}).get("litellm") or settings.get("model")}) diff --git a/hermit/agent/base/trajectory.py b/hermit/agent/base/trajectory.py index b33039af..664e2890 100644 --- a/hermit/agent/base/trajectory.py +++ b/hermit/agent/base/trajectory.py @@ -233,6 +233,8 @@ def mini_swe_agent_steps(stream: list[dict]) -> list[Step]: continue role, content = message.get("role"), str(message.get("content") or "") if role == "assistant": + if message.get("reasoning"): + steps.append(Step("reasoning", str(message["reasoning"]))) if content.strip(): steps.append(Step("message", content)) for command in message.get("actions") or []: diff --git a/tests/core/test_mini_swe_agent.py b/tests/core/test_mini_swe_agent.py index 97ab7716..cb12c94a 100644 --- a/tests/core/test_mini_swe_agent.py +++ b/tests/core/test_mini_swe_agent.py @@ -15,7 +15,10 @@ import sys import threading -from jinja2 import StrictUndefined, Template +try: + from jinja2 import StrictUndefined, Template +except ImportError: # mini-swe-agent brings jinja2: without either, the module is skipped (pytestmark) + StrictUndefined = Template = None import pytest import yaml @@ -298,6 +301,10 @@ def test_what_mini_swe_agent_cannot_do_is_refused(tmp_path, monkeypatch, scripte "model_class": "openrouter"}) with pytest.raises(Unsupported, match="calls its own service"): bound.run("x", workspace=tmp_path) + # without a binding too, a class whose messages or errors are not litellm's: usage, steps or retries unread + for model_class in ("litellm_response", "openrouter", "portkey_response"): + with pytest.raises(Unsupported, match="reads litellm and litellm_textbased only"): + agent.configured(model_class=model_class).run("x", workspace=tmp_path) # a config that names the model or the environment class fails every task: no session starts, and the acting # agent reports the error rather than score an empty answer server = scripted([]) @@ -385,6 +392,72 @@ def test_the_prompts_come_from_minis_package_not_a_mini_yaml_where_it_runs(tmp_p assert mini.base_config("litellm_textbased") == "mini_textbased" and mini.base_config("litellm") == "mini" +def test_the_runners_settings_file_names_no_host_path_as_pis_does(tmp_path): + """Under isolation: sandbox the run's sandbox translates the command, the environment and the prompt, not a file + in the home: the config is read into the settings and the system instructions go on the command.""" + config = tmp_path / "outside" / "mini.yaml" + config.parent.mkdir() + config.write_text("agent:\n step_limit: 7\n") + agent = make("mini-swe-agent", {"web_search": False, "model": "openai/m", "config": str(config), + "system": f"Read {tmp_path}/outside/notes.md first."}) + home = tmp_path / "home" + home.mkdir() + agent.prepare_home(home, (), None, "hermit_tools") + written = (home / "mini-swe-agent.json").read_text() + assert str(tmp_path) not in written and json.loads(written)["config"] == {"agent": {"step_limit": 7}} + command = agent.command("task", tmp_path / "work", home / "last_message.txt") + assert command[command.index("--system") + 1] == f"Read {tmp_path}/outside/notes.md first." + # past what one argument holds (Linux: 128 KiB), a file in the home, as Claude Code's + long = agent.configured(system="x" * 200_000).command("task", tmp_path / "work", home / "last_message.txt") + assert "--system" not in long and Path(long[long.index("--system-file") + 1]).read_text() == "x" * 200_000 + + +@needs_bwrap +@pytest.mark.parametrize("size", [0, 200_000], ids=["short", "past-one-argument"]) +def test_under_its_own_sandbox_the_base_agent_reads_its_config_and_instructions(tmp_path, monkeypatch, scripted, size): + """isolation: sandbox, as Pi's: the config outside the workspace (its step_limit of 1) and the system + instructions reach the session, which the run's sandbox shows only its workspace and home.""" + from hermit.sandbox import Sandbox + monkeypatch.setenv("HERMIT_TEST_KEY", KEY) + server = scripted([bash("echo 4 > answer.txt", "c1"), submit()]) + config = tmp_path / "outside" / "mini.yaml" + config.parent.mkdir() + config.write_text("agent:\n step_limit: 1\n") + work = tmp_path / "work" + work.mkdir() + settings = {**params(server, config=str(config)), "home": str(tmp_path / "home"), "isolation": "sandbox", + "timeout_s": 120, "sandbox": Sandbox(network="isolated", env_allow=[], writable=[work], role="agent", + record_dir=tmp_path / "records")} + system = "Answer with one digit." + "\n" * size + agent, prompt = make("mini-swe-agent", settings).instructed(system, "What is 2 + 2?") + result = agent.run(prompt, workspace=work) + assert result.extra["served"] and len(server.requests) == 1 # the config's step_limit + assert "Answer with one digit." in server.requests[0]["messages"][0]["content"] + [record] = (tmp_path / "records").glob("agent-*.json") + assert json.loads(record.read_text())["network"] == "isolated" + + +def test_a_thinking_models_reasoning_is_a_step_of_the_trajectory(): + """Codex's, Claude Code's and Pi's readers keep the model's reasoning as steps; mini-swe-agent's events carry + litellm's reasoning_content so that its reader does too.""" + from hermit.agent.base.mini_swe_agent_run import event_of + from hermit.agent.base.trajectory import mini_swe_agent_steps + event = event_of({"role": "assistant", "content": None, "reasoning_content": "The total is in column C.", + "extra": {"actions": [{"command": "cut -d, -f3 inventory.csv"}]}}) + steps = mini_swe_agent_steps([event]) + assert [(s.kind, s.text or s.command) for s in steps] == [("reasoning", "The total is in column C."), + ("command", "cut -d, -f3 inventory.csv")] + + +def test_after_a_format_error_mini_is_told_to_end_by_submitting_its_answer(): + """mini's own recovery message ends a task with the bare COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT, an empty + submission: Hermit's tells the model to submit answer.txt, as its instance template does.""" + template = mini.render_config("mini")["model"]["format_error_template"] + told = Template(template).render(error="no tool call", finish_reason="stop", has_tool_calls=False) + assert "`echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT && cat answer.txt`" in told + assert "COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`\n" not in told + + def test_the_layered_config_names_mini_swe_agent_and_binds_its_model(tmp_path): from hermit.config import Config path = make_task(tmp_path, agent={"base": "mini-swe-agent", "model": "deepseek/deepseek-flash",