From 8f0dba1c04cbfb284334f4b29f27fd86eb989113 Mon Sep 17 00:00:00 2001 From: Eric Lee Date: Mon, 13 Jul 2026 18:48:22 -0700 Subject: [PATCH] =?UTF-8?q?feat(eval):=20/eco=20token-compression=20benchm?= =?UTF-8?q?ark=20=E2=80=94=20measured,=20README=20showcase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replicate the experiment behind RTK's README savings table (https://github.com/rtk-ai/rtk estimates -80% over a modeled 30-minute session) with real measurements, and showcase the results in the READMEs. eval/eco/ benchmark: - capture_corpus.py (stdlib-only) captures 27 real operations: failing + green pytest/go/jest runs from sample projects with genuine bugs (RTK's "never synthetic" fixture rule), npm/pip installs, git clone/status/ commit/push --progress, repo-scale log/diff/ls/find/cat/grep, a ~35k-line macOS system log, and small must-pass-through rows. Marker-file guard refuses to rmtree a --workdir it didn't create; missing toolchains skip; partial failure exits non-zero. - measure.py replays each capture through the production path (baseline = _assemble_bash_body(truncate_output(...)); compress_bash_output; the mapper's returnCodeInterpretation suffix on both sides) and counts tiktoken cl100k_base tokens of the exact wire content. Tee-filename entropy (time_ns/pid/counter) is pinned and the tee dir fixed, so reruns on an unchanged corpus are byte-identical. Never-worse asserted corpus-wide. Percentages floor so display never overstates savings. - Committed results: corpus total 92,989 -> 17,767 tokens (-80%), filter hits -88%, and an honestly conservative recompute of RTK's own session model at -19% (eco compresses results only; small outputs pass through). READMEs: new /eco section (before/after pytest example, measured table, RTK-model comparison in
, safety guarantees, reproduce commands) in README.md and README_ZH.md, compact translated sections in AR/FR/HI/PT/ RU, /eco rows in the REPL command tables, eval/README.md pointer. src/eco: npm >=9 lowercased its log prefixes — noise pattern now matches "npm warn" alongside WARN/notice (npm error still always survives); found via the real npm-install capture. New regression test pins both casings. Critic-reviewed: two REVISE rounds (determinism + rmtree guard majors, wire-exactness and display-floor nits) then APPROVE with independent re-execution of the capture and byte-identical measure verification. Co-Authored-By: Claude Fable 5 --- .gitignore | 6 + README.md | 120 +++ docs/i18n/README_AR.md | 22 + docs/i18n/README_FR.md | 23 + docs/i18n/README_HI.md | 22 + docs/i18n/README_PT.md | 23 + docs/i18n/README_RU.md | 23 + docs/i18n/README_ZH.md | 88 ++ eval/README.md | 3 + eval/eco/README.md | 96 ++ eval/eco/capture_corpus.py | 919 ++++++++++++++++++ eval/eco/measure.py | 340 +++++++ .../examples/git-status-dirty.after.txt | 13 + .../examples/git-status-dirty.before.txt | 16 + .../results/examples/npm-install.after.txt | 1 + .../results/examples/npm-install.before.txt | 4 + .../results/examples/pytest-fail.after.txt | 38 + .../results/examples/pytest-fail.before.txt | 129 +++ eval/eco/results/results.json | 548 +++++++++++ eval/eco/results/results.md | 59 ++ src/eco/filters.py | 4 +- tests/test_eco.py | 17 + 22 files changed, 2513 insertions(+), 1 deletion(-) create mode 100644 eval/eco/README.md create mode 100644 eval/eco/capture_corpus.py create mode 100644 eval/eco/measure.py create mode 100644 eval/eco/results/examples/git-status-dirty.after.txt create mode 100644 eval/eco/results/examples/git-status-dirty.before.txt create mode 100644 eval/eco/results/examples/npm-install.after.txt create mode 100644 eval/eco/results/examples/npm-install.before.txt create mode 100644 eval/eco/results/examples/pytest-fail.after.txt create mode 100644 eval/eco/results/examples/pytest-fail.before.txt create mode 100644 eval/eco/results/results.json create mode 100644 eval/eco/results/results.md diff --git a/.gitignore b/.gitignore index e5319ba2d..7cd77f554 100644 --- a/.gitignore +++ b/.gitignore @@ -139,6 +139,12 @@ out/ /eval/runs/ /eval/__pycache__/ +# /eco benchmark — captured command outputs are machine-specific (and the +# macOS `log show` capture carries local system-log lines); regenerate with +# eval/eco/capture_corpus.py. The measured results/ tables ARE tracked. +/eval/eco/corpus/ +/eval/eco/__pycache__/ + # Stray *.patch / *.diff artifacts the agent occasionally writes via its # Write tool during SWE-bench eval (we disable Write/Edit in the wrapper # now but old artifacts may persist). diff --git a/README.md b/README.md index 193e25220..75b09de1e 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,125 @@ Reproduce locally — see [`eval/README.md`](eval/README.md) for the full workfl *** +## 🌿 `/eco` Token Compression — **-80% Bash-output tokens, measured** + +Long agentic sessions drown in tool output: failing test logs, `git` progress spam, +2,000-line listings. Toggle **`/eco`** on and ClawCodex compresses the *model-bound +rendering* of every Bash result with deterministic filters ported from +[RTK](https://github.com/rtk-ai/rtk)'s method set — failure-focused test summaries, +command-scoped ceremony stripping, log deduplication with `[×N]` counts, and a +recoverable head-cap — while the full raw output stays on disk behind a runnable +recovery hint. No model in the loop, no command rewriting, nothing to learn. + +```text +$ pytest # 128 lines → 37 lines, 1,347 → 390 tokens (-71%) +Pytest: 5 failed, 29 passed in 0.04s + +1. [FAIL] test_unknown_sku_message + with pytest.raises(OrderError, match="unknown sku 'gold-bar'"): + > o.total() + tests/test_orders.py:34: + ⋮ +5. [FAIL] test_truncate_one + > assert truncate_words("alpha beta", 1) == "alpha..." + E AssertionError: assert 'alpha beta...' == 'alpha...' +[full output: ~/.clawcodex///eco/1707_pytest.log] +Command failed with exit code 1 +``` + +**Measured, not estimated.** RTK's README models a 30-minute session and *estimates* +-80%. We ran the experiment instead: a 27-operation corpus of **real command +outputs** — failing `pytest`/`go test`/`jest` runs, `pip`/`npm` installs, git +workflows, repo-scale listings, a 34,000-line system log, captured live (RTK's own +"never synthetic" fixture rule) — replayed through the exact production pipeline, +counting tiktoken `cl100k_base` tokens of the model-bound text with `/eco` off vs on: + +| Operation | Filter | Raw tokens | `/eco` tokens | Saved | +|---|---|---:|---:|---:| +| `pytest` (failing run) | failure focus | 1,347 | 390 | **-71%** | +| `pytest -v` (failing run) | failure focus | 1,925 | 392 | **-79%** | +| `pytest -v` (green run) | one-line collapse | 359 | 60 | **-83%** | +| `go test -v ./...` (failing run) | failure focus | 527 | 227 | **-56%** | +| `npx jest` (failing run) | failure focus | 444 | 175 | **-60%** | +| `npm install jest` | ceremony strip | 188 | 8 | **-95%** | +| `pip install flask` | ceremony strip | 514 | 85 | **-83%** | +| `git clone --progress` | ceremony strip | 6,868 | 18 | **-99%** | +| `git push --progress` | ceremony strip | 6,458 | 75 | **-98%** | +| `git status` (dirty tree) | advice strip | 143 | 91 | **-36%** | +| `git log -n 300` | recoverable head-cap | 7,714 | 946 | **-87%** | +| `git diff v1.0.0..v1.1.0 -- src` | recoverable head-cap | 7,561 | 748 | **-90%** | +| `ls -R src` | recoverable head-cap | 9,088 | 225 | **-97%** | +| `cat` (900-line file) | recoverable head-cap | 6,833 | 552 | **-91%** | +| `grep -rn 'def ' src/` | recoverable head-cap | 7,582 | 1,219 | **-83%** | +| `log show --last 90s` (34k lines) | log dedup | 10,512 | 1,977 | **-81%** | +| **Whole corpus (27 operations)** | | **92,989** | **17,767** | **-80%** | + +The corpus also includes 8 operations that (correctly) pass through **byte-identical** +— a clean `git status`, `docker ps`, `ruff check` findings, a small failing `go test`, +a 370-line `grep` that sits under the head-cap threshold — because `/eco` guarantees +**never worse**: a compression that doesn't beat the raw rendering is discarded. +Full tables: [`eval/eco/results/results.md`](eval/eco/results/results.md). + +
+vs RTK's own 30-minute-session model (why our headline is honest) + +
+ +RTK *rewrites commands* into its own CLI (`rtk ls`, `rtk read`, `rtk grep`), so every +operation in its session model compresses. `/eco` deliberately compresses **results +only** — the command the model wrote is the command that runs — and small outputs pass +through untouched. Recomputing RTK's session table with our *measured* ratios (0% +where our corpus shows passthrough at RTK's assumed sizes): + +| Operation | Freq | Standard | rtk (estimated) | clawcodex `/eco` (measured) | +|---|---:|---:|---:|---:| +| `ls` / `tree` | 10x | 2,000 | 400 | 2,000 (0%) | +| `cat` / read | 20x | 40,000 | 12,000 | 40,000 (0%) | +| `grep` / `rg` | 8x | 16,000 | 3,200 | 16,000 (0%) | +| `git status` | 10x | 3,000 | 600 | 1,908 (-36%) | +| `git diff` | 5x | 10,000 | 2,500 | 10,000 (0%) | +| `git log` | 5x | 2,500 | 500 | 2,500 (0%) | +| `git add/commit/push` | 8x | 1,600 | 120 | 1,007 (-37%) | +| `cargo test` / `npm test` | 5x | 25,000 | 2,500 | 9,850 (-60%) | +| `ruff check` | 3x | 3,000 | 600 | 3,000 (0%) | +| `pytest` | 4x | 8,000 | 800 | 2,320 (-71%) | +| `go test` | 3x | 6,000 | 600 | 6,000 (0%) | +| `docker ps` | 3x | 900 | 180 | 900 (0%) | +| **Total** | | **~118,000** | **~23,900 (-80%)** | **~95,500 (-19%)** | + +Under RTK's *averaged* assumptions (every `cat` ≈ 2,000 tokens, every `ls` ≈ 200) the +honest number for a results-only compressor is **-19%** — those mid-size outputs are +exactly what ClawCodex already handles with Read-tool line caps and 30k-char Bash +truncation. But real sessions aren't averages: they're fat-tailed, and one 2,000-line +`git log`, one failing suite, or one `npm install` blows more context than fifty small +commands. `/eco` targets precisely that tail — which is why the measured number on +real outputs is **-80%**, the same figure RTK estimates, with none of the risk of +rewriting commands. + +
+ +**The RTK safety rules, kept** (see [`src/eco/`](src/eco/)): + +- **Never worse** — every compressed rendering is token-checked against the exact + baseline it replaces; the baseline wins ties. Worst case is 0% saved, never negative. +- **Failures survive** — error/failure lines are never rewritten, only ceremony drops; + a green summary with a non-zero exit code is treated as untrusted and passed through. +- **Everything recoverable** — lossy compressions tee the full output to the session + dir and append a runnable hint (`[see remaining: tail -n +61 …]`); no tee, no compression. +- **Semantics untouched** — exit codes, `is_error`, images, background tasks, and + interrupted runs are never altered; any filter exception falls back to passthrough. + +`/eco status` shows per-filter savings for the session. Compression stacks with the +[DeepSeek prefix cache](#-deepseek-prefix-cache): the cache makes the stable prefix +nearly free, `/eco` shrinks the fresh suffix every turn actually pays for. Reproduce: + +```bash +python3 eval/eco/capture_corpus.py --workdir /tmp/eco-bench # capture real outputs +.venv/bin/python eval/eco/measure.py # replay + count tokens +``` + +*** + ## ⭐ Star History [View star history on star-history.com](https://www.star-history.com/?repos=agentforce314%2Fclawcodex&type=date&legend=top-left) @@ -476,6 +595,7 @@ clawcodex --help # All flags: -p, --provider, --model, … | `/skill` | Skill launcher flow | | `/context` | Workspace / prompt context (when available) | | `/compact` | Compact or clear conversation (fallback clears if compact unavailable) | +| `/eco` | Toggle Bash-output token compression (`on` / `off` / `status` for per-filter savings) | | `/exit`, `/quit`, `/q` | Exit | ### Skills (Slash Commands) diff --git a/docs/i18n/README_AR.md b/docs/i18n/README_AR.md index 6d50fc53c..b5b3863e5 100644 --- a/docs/i18n/README_AR.md +++ b/docs/i18n/README_AR.md @@ -57,6 +57,28 @@ *** +## 🌿 ضغط التوكنات `/eco` — **‏-80% مُقاسة على مخرجات Bash** + +فعّل **`/eco`** وسيضغط ClawCodex العرض المُرسل إلى النموذج لكل نتيجة Bash عبر +مرشّحات حتمية منقولة من [RTK](https://github.com/rtk-ai/rtk): ملخّصات اختبارات تركّز +على الإخفاقات، إزالة الضجيج الشكلي (`git`/`pip`/`npm`)، إزالة تكرار السجلات، وقصّ +قابل للاسترجاع — بينما يبقى الناتج الخام كاملًا على القرص خلف تلميح قابل للتنفيذ +مباشرة. ضمان **لا أسوأ أبدًا**: أي ضغط لا يتفوّق على العرض الخام يُستبعد، وأسطر +الأخطاء تبقى دائمًا. + +مُقاس (توكنات tiktoken ‏`cl100k_base`) على 27 عملية حقيقية أُعيد تشغيلها عبر خطّ +الإنتاج: + +| العملية | خام | `/eco` | التوفير | +|---|---:|---:|---:| +| `pytest` (مع إخفاقات) | 1,347 | 390 | **-71%** | +| `git clone --progress` | 6,868 | 18 | **-99%** | +| `ls -R src` | 9,088 | 225 | **-97%** | +| `log show --last 90s` (34 ألف سطر) | 10,512 | 1,977 | **-81%** | +| **كامل المجموعة (27 عملية)** | **92,989** | **17,767** | **-80%** | + +الجداول الكاملة والمنهجية: [`eval/eco/`](../../eval/eco/README.md). + ## ⭐ Star History diff --git a/docs/i18n/README_FR.md b/docs/i18n/README_FR.md index e0fe0927a..df0858fdb 100644 --- a/docs/i18n/README_FR.md +++ b/docs/i18n/README_FR.md @@ -57,6 +57,29 @@ *** +## 🌿 Compression de tokens `/eco` — **-80 % mesurés sur la sortie Bash** + +Activez **`/eco`** et ClawCodex compresse le rendu côté modèle de chaque résultat Bash +avec des filtres déterministes portés de [RTK](https://github.com/rtk-ai/rtk) : résumés +de tests focalisés sur les échecs, suppression du cérémonial (`git`/`pip`/`npm`), +déduplication de logs et troncature récupérable — la sortie brute complète reste sur +disque derrière un indice directement exécutable. Garantie **jamais pire** : une +compression qui ne bat pas le rendu brut est abandonnée, et les lignes d'erreur +survivent toujours. + +Mesuré (tokens tiktoken `cl100k_base`) sur 27 opérations réelles rejouées à travers le +pipeline de production : + +| Opération | Brut | `/eco` | Gain | +|---|---:|---:|---:| +| `pytest` (échecs) | 1,347 | 390 | **-71%** | +| `git clone --progress` | 6,868 | 18 | **-99%** | +| `ls -R src` | 9,088 | 225 | **-97%** | +| `log show --last 90s` (34k lignes) | 10,512 | 1,977 | **-81%** | +| **Corpus complet (27 opérations)** | **92,989** | **17,767** | **-80%** | + +Tables complètes et méthodologie : [`eval/eco/`](../../eval/eco/README.md). + ## ⭐ Star History diff --git a/docs/i18n/README_HI.md b/docs/i18n/README_HI.md index f4a813cfc..a76538b19 100644 --- a/docs/i18n/README_HI.md +++ b/docs/i18n/README_HI.md @@ -57,6 +57,28 @@ *** +## 🌿 `/eco` टोकन संपीड़न — **Bash आउटपुट पर मापा गया -80%** + +**`/eco`** चालू करें और ClawCodex हर Bash परिणाम की मॉडल-बाउंड रेंडरिंग को +[RTK](https://github.com/rtk-ai/rtk) से पोर्ट किए गए नियतात्मक फ़िल्टरों से संपीड़ित +करता है: विफलता-केंद्रित टेस्ट सारांश, औपचारिक शोर हटाना (`git`/`pip`/`npm`), लॉग +डीडुप्लीकेशन और पुनर्प्राप्ति-योग्य ट्रंकेशन — पूरा कच्चा आउटपुट एक चलाने-योग्य संकेत के +पीछे डिस्क पर सुरक्षित रहता है। **कभी बदतर नहीं** की गारंटी: जो संपीड़न कच्ची रेंडरिंग से +बेहतर नहीं, वह छोड़ दिया जाता है, और त्रुटि पंक्तियाँ हमेशा बची रहती हैं। + +उत्पादन पाइपलाइन से दोहराए गए 27 वास्तविक ऑपरेशनों पर मापा गया (tiktoken +`cl100k_base` टोकन): + +| ऑपरेशन | कच्चा | `/eco` | बचत | +|---|---:|---:|---:| +| `pytest` (विफलताओं सहित) | 1,347 | 390 | **-71%** | +| `git clone --progress` | 6,868 | 18 | **-99%** | +| `ls -R src` | 9,088 | 225 | **-97%** | +| `log show --last 90s` (34k पंक्तियाँ) | 10,512 | 1,977 | **-81%** | +| **पूरा कॉर्पस (27 ऑपरेशन)** | **92,989** | **17,767** | **-80%** | + +पूर्ण तालिकाएँ और कार्यप्रणाली: [`eval/eco/`](../../eval/eco/README.md)। + ## ⭐ Star History diff --git a/docs/i18n/README_PT.md b/docs/i18n/README_PT.md index df6de3f5a..9adb333b1 100644 --- a/docs/i18n/README_PT.md +++ b/docs/i18n/README_PT.md @@ -57,6 +57,29 @@ *** +## 🌿 Compressão de tokens `/eco` — **-80% medidos na saída do Bash** + +Ative **`/eco`** e o ClawCodex comprime a renderização enviada ao modelo de cada +resultado do Bash com filtros determinísticos portados do +[RTK](https://github.com/rtk-ai/rtk): resumos de testes focados nas falhas, remoção de +cerimônia (`git`/`pip`/`npm`), deduplicação de logs e truncamento recuperável — a saída +bruta completa fica em disco atrás de uma dica executável. Garantia **nunca pior**: uma +compressão que não supera a renderização bruta é descartada, e linhas de erro sempre +sobrevivem. + +Medido (tokens tiktoken `cl100k_base`) em 27 operações reais reproduzidas pelo pipeline +de produção: + +| Operação | Bruto | `/eco` | Ganho | +|---|---:|---:|---:| +| `pytest` (com falhas) | 1,347 | 390 | **-71%** | +| `git clone --progress` | 6,868 | 18 | **-99%** | +| `ls -R src` | 9,088 | 225 | **-97%** | +| `log show --last 90s` (34k linhas) | 10,512 | 1,977 | **-81%** | +| **Corpus completo (27 operações)** | **92,989** | **17,767** | **-80%** | + +Tabelas completas e metodologia: [`eval/eco/`](../../eval/eco/README.md). + ## ⭐ Star History diff --git a/docs/i18n/README_RU.md b/docs/i18n/README_RU.md index 685d30778..1859592c6 100644 --- a/docs/i18n/README_RU.md +++ b/docs/i18n/README_RU.md @@ -57,6 +57,29 @@ *** +## 🌿 Сжатие токенов `/eco` — **-80% на выводе Bash, измерено** + +Включите **`/eco`**, и ClawCodex сжимает отправляемое модели представление каждого +результата Bash детерминированными фильтрами, портированными из +[RTK](https://github.com/rtk-ai/rtk): сводки тестов с фокусом на падениях, срезание +церемониального шума (`git`/`pip`/`npm`), дедупликация логов и восстановимое усечение — +полный сырой вывод остаётся на диске за исполняемой подсказкой. Гарантия **никогда не +хуже**: сжатие, не превосходящее сырой вывод, отбрасывается, а строки ошибок всегда +сохраняются. + +Измерено (токены tiktoken `cl100k_base`) на 27 реальных операциях, проигранных через +производственный конвейер: + +| Операция | Сырой | `/eco` | Экономия | +|---|---:|---:|---:| +| `pytest` (с падениями) | 1,347 | 390 | **-71%** | +| `git clone --progress` | 6,868 | 18 | **-99%** | +| `ls -R src` | 9,088 | 225 | **-97%** | +| `log show --last 90s` (34k строк) | 10,512 | 1,977 | **-81%** | +| **Весь корпус (27 операций)** | **92,989** | **17,767** | **-80%** | + +Полные таблицы и методология: [`eval/eco/`](../../eval/eco/README.md). + ## ⭐ Star History diff --git a/docs/i18n/README_ZH.md b/docs/i18n/README_ZH.md index b5866bbb5..4e7c11572 100644 --- a/docs/i18n/README_ZH.md +++ b/docs/i18n/README_ZH.md @@ -110,6 +110,93 @@ python -m src.cli --dangerously-skip-permissions # 启动 REPL *** +## 🌿 `/eco` Token 压缩 —— **Bash 输出 token 实测 -80%** + +长时间的 agentic 会话会被工具输出淹没：失败的测试日志、`git` 进度刷屏、2,000 行的目录列表。打开 **`/eco`** 后，ClawCodex 会用一组从 [RTK](https://github.com/rtk-ai/rtk) 方法集移植的确定性过滤器压缩每个 Bash 结果的*模型侧渲染* —— 聚焦失败的测试摘要、按命令族裁剪仪式性输出、带 `[×N]` 计数的日志去重、可恢复的头部截断 —— 完整原始输出则保留在磁盘上，并附带一条可直接运行的恢复提示。不经过模型、不改写命令、无需学习成本。 + +```text +$ pytest # 128 行 → 37 行，1,347 → 390 tokens（-71%） +Pytest: 5 failed, 29 passed in 0.04s + +1. [FAIL] test_unknown_sku_message + with pytest.raises(OrderError, match="unknown sku 'gold-bar'"): + > o.total() + tests/test_orders.py:34: + ⋮ +5. [FAIL] test_truncate_one + > assert truncate_words("alpha beta", 1) == "alpha..." + E AssertionError: assert 'alpha beta...' == 'alpha...' +[full output: ~/.clawcodex///eco/1707_pytest.log] +Command failed with exit code 1 +``` + +**实测，而非估算。** RTK 的 README 对一个 30 分钟会话做建模并*估算*出 -80%。我们直接做了实验：一个由 27 个操作组成的**真实命令输出**语料 —— 失败的 `pytest`/`go test`/`jest` 运行、`pip`/`npm` 安装、git 工作流、仓库级列举、34,000 行系统日志，全部现场捕获（遵循 RTK 自己的 "never synthetic" 夹具规则）—— 经由生产管线逐字节重放，用 tiktoken `cl100k_base` 统计 `/eco` 关闭与开启时模型侧文本的 token 数： + +| 操作 | 过滤器 | 原始 tokens | `/eco` tokens | 节省 | +|---|---|---:|---:|---:| +| `pytest`（失败运行） | 失败聚焦 | 1,347 | 390 | **-71%** | +| `pytest -v`（失败运行） | 失败聚焦 | 1,925 | 392 | **-79%** | +| `pytest -v`（全绿运行） | 单行折叠 | 359 | 60 | **-83%** | +| `go test -v ./...`（失败运行） | 失败聚焦 | 527 | 227 | **-56%** | +| `npx jest`（失败运行） | 失败聚焦 | 444 | 175 | **-60%** | +| `npm install jest` | 仪式裁剪 | 188 | 8 | **-95%** | +| `pip install flask` | 仪式裁剪 | 514 | 85 | **-83%** | +| `git clone --progress` | 仪式裁剪 | 6,868 | 18 | **-99%** | +| `git push --progress` | 仪式裁剪 | 6,458 | 75 | **-98%** | +| `git status`（脏工作区） | 建议行裁剪 | 143 | 91 | **-36%** | +| `git log -n 300` | 可恢复头部截断 | 7,714 | 946 | **-87%** | +| `git diff v1.0.0..v1.1.0 -- src` | 可恢复头部截断 | 7,561 | 748 | **-90%** | +| `ls -R src` | 可恢复头部截断 | 9,088 | 225 | **-97%** | +| `cat`（900 行源文件） | 可恢复头部截断 | 6,833 | 552 | **-91%** | +| `grep -rn 'def ' src/` | 可恢复头部截断 | 7,582 | 1,219 | **-83%** | +| `log show --last 90s`（34k 行） | 日志去重 | 10,512 | 1,977 | **-81%** | +| **整个语料（27 个操作）** | | **92,989** | **17,767** | **-80%** | + +语料中还有 8 个操作（正确地）**逐字节原样通过** —— 干净的 `git status`、`docker ps`、`ruff check` 的告警、小规模失败的 `go test`、一个位于头部截断阈值之下的 370 行 `grep` —— 因为 `/eco` 保证**绝不更差**：压不过原始渲染的结果直接弃用。完整表格见 [`eval/eco/results/results.md`](../../eval/eco/results/results.md)。 + +
+对照 RTK 自己的 30 分钟会话模型（为什么我们的标题数字是诚实的） + +
+ +RTK 会把命令*改写*成它自己的 CLI（`rtk ls`、`rtk read`、`rtk grep`），所以其会话模型中每一行都在压缩。`/eco` 刻意**只压缩结果** —— 模型写的命令就是实际运行的命令 —— 小输出原样通过。用我们的*实测*比例重算 RTK 的会话表（在 RTK 假设的尺寸下语料显示为原样通过的行记 0%）： + +| 操作 | 频次 | 标准 | rtk（估算） | clawcodex `/eco`（实测） | +|---|---:|---:|---:|---:| +| `ls` / `tree` | 10x | 2,000 | 400 | 2,000（0%） | +| `cat` / read | 20x | 40,000 | 12,000 | 40,000（0%） | +| `grep` / `rg` | 8x | 16,000 | 3,200 | 16,000（0%） | +| `git status` | 10x | 3,000 | 600 | 1,908（-36%） | +| `git diff` | 5x | 10,000 | 2,500 | 10,000（0%） | +| `git log` | 5x | 2,500 | 500 | 2,500（0%） | +| `git add/commit/push` | 8x | 1,600 | 120 | 1,007（-37%） | +| `cargo test` / `npm test` | 5x | 25,000 | 2,500 | 9,850（-60%） | +| `ruff check` | 3x | 3,000 | 600 | 3,000（0%） | +| `pytest` | 4x | 8,000 | 800 | 2,320（-71%） | +| `go test` | 3x | 6,000 | 600 | 6,000（0%） | +| `docker ps` | 3x | 900 | 180 | 900（0%） | +| **合计** | | **~118,000** | **~23,900（-80%）** | **~95,500（-19%）** | + +在 RTK 的*平均化*假设下（每个 `cat` ≈ 2,000 tokens、每个 `ls` ≈ 200），只压结果的诚实数字是 **-19%** —— 这些中等尺寸的输出恰好是 ClawCodex 已经用 Read 工具行数上限与 30k 字符 Bash 截断处理掉的部分。但真实会话不是平均值：它是重尾分布，一次 2,000 行的 `git log`、一轮失败的测试套件或一次 `npm install` 烧掉的上下文超过五十条小命令。`/eco` 瞄准的正是这条尾巴 —— 所以在真实输出上的实测数字是 **-80%**，与 RTK 估算的数字相同，却没有任何改写命令的风险。 + +
+ +**RTK 的安全规则全部保留**（见 [`src/eco/`](../../src/eco/)）： + +- **绝不更差** —— 每个压缩渲染都会与其替换的精确基线做 token 比对；平局时基线获胜。最坏情况是节省 0%，绝不为负。 +- **失败信息幸存** —— 错误/失败行从不被改写，只裁剪仪式性输出；绿色摘要 + 非零退出码被视为不可信并原样通过。 +- **一切可恢复** —— 有损压缩会把完整输出 tee 到会话目录并附加可运行的提示（`[see remaining: tail -n +61 …]`）；写不了 tee 就不压缩。 +- **语义不动** —— 退出码、`is_error`、图像、后台任务与中断运行一概不改；过滤器抛出任何异常都会回退为原样通过。 + +`/eco status` 展示本会话的分过滤器节省。压缩与 **DeepSeek 前缀缓存**（见[新闻](#-新闻) 2026-06-18）叠加：缓存让稳定前缀近乎免费，`/eco` 则缩小每轮真正要付费的新增后缀。复现： + +```bash +python3 eval/eco/capture_corpus.py --workdir /tmp/eco-bench # 捕获真实输出 +.venv/bin/python eval/eco/measure.py # 重放并统计 token +``` + +*** + ## ⭐ Star 历史 [在 star-history.com 查看 Star 历史图表](https://www.star-history.com/?repos=agentforce314%2Fclawcodex&type=date&legend=top-left) @@ -394,6 +481,7 @@ clawcodex --help # 全部参数：-p、--provider、--model 等 | `/skill` | 技能启动流程 | | `/context` | 工作区 / 提示上下文（若可用） | | `/compact` | 压缩或清空对话（不可用时回退为清空） | +| `/eco` | 切换 Bash 输出 token 压缩（`on` / `off` / `status` 查看分过滤器节省） | | `/exit`、`/quit`、`/q` | 退出 | ### Skills（技能 / 斜杠命令） diff --git a/eval/README.md b/eval/README.md index e05f247e0..d948de902 100644 --- a/eval/README.md +++ b/eval/README.md @@ -1,5 +1,8 @@ # Eval — clawcodex vs openclaude on SWE-bench +> Sibling benchmark: [`eval/eco/`](eco/README.md) measures the `/eco` +> token-compression savings on real command outputs. + This directory drives a side-by-side comparison of **clawcodex** (this repo) and **openclaude** (the TypeScript reference) against [SWE-bench](https://swe-bench.github.io). The goal is to confirm parity: diff --git a/eval/eco/README.md b/eval/eco/README.md new file mode 100644 index 000000000..b7c01788f --- /dev/null +++ b/eval/eco/README.md @@ -0,0 +1,96 @@ +# Eval — `/eco` token-compression benchmark + +This directory reproduces, with **measurements**, the experiment behind +[RTK](https://github.com/rtk-ai/rtk)'s README savings table. RTK models a +30-minute agent session and *estimates* per-operation token savings +("Estimates based on medium-sized TypeScript/Rust projects"). Here the same +class of operations is **captured live and counted**: every number in the +ClawCodex README's `/eco` section comes from replaying real command outputs +through the production compression pipeline. + +## Layout + +```text +eval/eco/ +├── capture_corpus.py # runs real commands, stores full outputs (stdlib-only) +├── measure.py # replays corpus through src/eco, counts tokens +├── corpus/ # gitignored — machine-specific captured outputs +└── results/ # committed — results.md, results.json, examples/ +``` + +## Method + +1. **Capture** (`capture_corpus.py`). 27 operations an agent actually runs: + - *Test runners*: `pytest` / `pytest -v` (failing + green), `go test [-v] ./...` + (failing), `npx jest` (failing + green), plus a green slice of this repo's own + suite. Failing runs come from small sample projects written into `--workdir` + with genuine bugs (off-by-one, in-place mutation, wrong error message) so the + tracebacks, `--- FAIL` blocks, and assertion diffs are real tool output — + RTK's own "never synthetic" fixture rule. + - *Package managers*: `npm install jest`, `pip install flask` into a fresh venv + (`--no-cache-dir` so the Collecting/Downloading ceremony is real). + - *Git*: `clone --no-local --progress` of this repo, dirty + clean `git status`, + `commit`, `push --progress` to a local bare remote. + - *Repo-scale reads*: `git log` (full + oneline), `git diff v1.0.0..v1.1.0`, + `ls -R src`, `find`, `cat` a 900-line file, two `grep -rn` sweeps (one under, + one over the 400-line head-cap threshold). + - *Logs*: `log show --last 90s --style syslog` (macOS unified log; ~35k + lines, varies by run). + - *Small outputs*: `docker ps`, `wc -l`, clean `git status` — passthrough rows + that must come back byte-identical. + +2. **Measure** (`measure.py`). Each item is replayed through the *production* + path, byte-for-byte what the Bash tool does when `/eco` is on: + + ```python + baseline = _assemble_bash_body(truncate_output(stdout), truncate_output(stderr)) + outcome = compress_bash_output(command, exit_code, full_text, baseline, tee_dir) + ``` + + `baseline` is exactly the model-bound text without eco (including the + pre-existing 30k-char truncation), and both renderings get the mapper's + `returnCodeInterpretation` suffix, so the counted text is the wire content. + The eco text includes the recovery-hint line, so its token cost is charged + honestly — for the committed run, the bench tee path tokenizes no *shorter* + than a typical production `~/.clawcodex/...` hint, so its published savings + are, if anything, slightly understated. Tokens are tiktoken `cl100k_base` counts (RTK + reports chars/4 estimates; we compute those too, into `results.json`, but + headline numbers are real tokenizer counts). The never-worse invariant is + asserted over the whole corpus, and tee-filename entropy (`time_ns`, pid) + is pinned so reruns on an unchanged corpus are byte-identical. + +3. **Project** — `results.md` also recomputes RTK's 30-minute-session table + with our measured ratios: rows whose assumed per-call size sits below eco's + thresholds (a ~200-token `ls`, a ~2,000-token `cat`) map to 0% because eco + passes them through; `git add/commit/push` applies the measured push ratio + to the push share (3 of 8 calls) only. That projection is deliberately + conservative — the corpus table is the measured result. + +## Reproduce + +```bash +python3 eval/eco/capture_corpus.py --workdir /tmp/eco-bench +.venv/bin/python eval/eco/measure.py # needs tiktoken (repo venv has it) +``` + +`capture_corpus.py` needs `git`, and captures what it can find: `go`, `npm`, +`docker` (daemon up), and macOS `log` rows are skipped when unavailable. +`--python` points at a python that can `import pytest` (defaults to the repo +venv). Outputs are machine-specific — sizes and therefore percentages will +vary a few points from the committed `results/` run; the invariants (never +worse, failures survive) hold everywhere. + +## Caveats, honestly + +- The corpus is **fat-tail weighted** by design: it includes the big outputs + (repo-scale listings, a failing suite, a system log) that dominate real + context spend, alongside 8 passthrough rows. The per-operation table is the + primary result; the -80% corpus total describes this mix, not every session. +- The `git clone` / `git push` rows are captured **with `--progress`** (as + agents often run them for visibility); a plain non-TTY push emits far less + ceremony. Those two rows contribute ~13k baseline tokens — dropping both + still leaves the corpus total around -77%. +- `cargo test` isn't captured (no local Rust toolchain); the RTK-model + projection maps its `cargo test / npm test` row to the measured jest ratio. +- Percentages for the same operation differ run to run (system-log volume, + npm registry noise, repo growth) — regenerate before quoting new numbers. diff --git a/eval/eco/capture_corpus.py b/eval/eco/capture_corpus.py new file mode 100644 index 000000000..8494ae825 --- /dev/null +++ b/eval/eco/capture_corpus.py @@ -0,0 +1,919 @@ +#!/usr/bin/env python3 +"""Capture a real-output corpus for the /eco token-compression benchmark. + +Mirrors the experiment behind RTK's README savings table +(https://github.com/rtk-ai/rtk): a set of operations an agent actually runs +in a coding session — test runners, git, package installs, listings, logs — +except every number here is *measured from a real command run*, never +estimated. This script only captures; ``measure.py`` replays the captures +through the production eco pipeline and counts tokens. + +Corpus item = one JSON file: the command an agent would type, its exit code, +and full (untruncated) stdout/stderr. Sample projects (pytest/go/jest with +genuine failing tests) are built in ``--workdir`` so the failure output is +real tool output, RTK's own fixture rule ("never synthetic"). Repo-scale +items (git log/diff, big listings) run against this repository itself. + +Stdlib-only on purpose: capture must not depend on the package under test. + +Usage: + python3 eval/eco/capture_corpus.py --workdir /tmp/eco-bench + # then: .venv/bin/python eval/eco/measure.py +""" + +from __future__ import annotations + +import argparse +import datetime as _dt +import json +import platform +import shutil +import subprocess +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +CORPUS_DIR = Path(__file__).resolve().parent / "corpus" + +# Full outputs are stored untruncated up to this cap; the eco engine itself +# accepts up to 10 MiB, and nothing in this corpus should get near either. +MAX_STORE_BYTES = 5 * 1024 * 1024 + +# Dropped into --workdir on creation; a pre-existing non-empty dir without it +# is refused rather than rmtree'd (a mistyped path must not cost user data). +WORKDIR_MARKER = ".eco-bench-workdir" + + +def _now_iso() -> str: + return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds") + + +def _pytest_python(cli_override: str | None) -> str: + """A python that can actually run pytest: --python, repo venv, this one.""" + for cand in (cli_override, str(REPO_ROOT / ".venv/bin/python"), sys.executable): + if not cand or not Path(cand).exists(): + continue + try: + probe = subprocess.run( + [cand, "-c", "import pytest"], capture_output=True, timeout=60 + ) + except (OSError, subprocess.TimeoutExpired): + continue + if probe.returncode == 0: + return cand + raise SystemExit( + "no python with pytest importable — pass --python /path/to/venv/bin/python" + ) + + +class Capture: + def __init__(self, corpus_dir: Path) -> None: + self.corpus_dir = corpus_dir + self.corpus_dir.mkdir(parents=True, exist_ok=True) + self.ok: list[str] = [] + self.failed: list[str] = [] + + def run( + self, + label: str, + op: str, + command: str, + *, + cwd: Path, + executed: str | None = None, + timeout: int = 180, + category: str = "", + ) -> None: + """Run one corpus command and store its full output. + + ``command`` is the agent-facing string (what eco's command-family + matchers see); ``executed`` overrides what is actually run (e.g. a + venv-qualified pytest) and defaults to ``command``. + """ + try: + t0 = time.monotonic() + proc = subprocess.run( + executed or command, + shell=True, + cwd=str(cwd), + capture_output=True, + text=True, + errors="replace", + timeout=timeout, + ) + dur_ms = int((time.monotonic() - t0) * 1000) + item = { + "label": label, + "op": op, + "category": category, + "command": command, + "executed": executed or command, + "exit_code": proc.returncode, + "stdout": proc.stdout[:MAX_STORE_BYTES], + "stderr": proc.stderr[:MAX_STORE_BYTES], + "duration_ms": dur_ms, + "captured_at": _now_iso(), + "platform": platform.platform(), + } + out = self.corpus_dir / f"{label}.json" + out.write_text(json.dumps(item, ensure_ascii=False, indent=1)) + n_lines = item["stdout"].count("\n") + item["stderr"].count("\n") + print(f" [ok] {label}: exit={proc.returncode} ~{n_lines} lines {dur_ms}ms") + self.ok.append(label) + except Exception as exc: # noqa: BLE001 — one bad item must not kill the sweep + print(f" [SKIP] {label}: {exc}") + self.failed.append(label) + + +# ── sample projects (real tools, genuine failing tests) ───────────────────── + + +def build_pysample(root: Path) -> Path: + """Small python lib + pytest suite; 5 of 34 tests fail on real bugs.""" + d = root / "pysample" + (d / "tests").mkdir(parents=True, exist_ok=True) + (d / "cart.py").write_text( + '''"""Tiny shopping-cart lib (sample project for the eco benchmark).""" +import os + + +class Cart: + def __init__(self): + self.items = [] + + def add(self, name, unit_price, qty=1): + if qty < 1: + raise ValueError("qty must be >= 1") + self.items.append((name, unit_price, qty)) + + def subtotal(self): + return sum(price * qty for _, price, qty in self.items) + + def total(self, discount_pct=0): + # BUG: discount is truncated to int cents per item instead of + # applied once to the subtotal — off by cents in real carts. + total = 0 + for _, price, qty in self.items: + line = price * qty + total += int(line * (100 - discount_pct)) / 100 + return total + + def count(self): + return sum(qty for _, _, qty in self.items) +''' + ) + (d / "textutil.py").write_text( + '''"""Text helpers (sample project for the eco benchmark).""" + + +def slugify(text): + out = [] + for ch in text.lower().strip(): + if ch.isalnum(): + out.append(ch) + elif out and out[-1] != "-": + out.append("-") + return "".join(out).strip("-") + + +def truncate_words(text, max_words): + words = text.split() + word_count = len(words) + if len(words) <= max_words: + return text + # BUG: keeps one word too many (classic off-by-one). + return " ".join(words[: max_words + 1]) + "..." +''' + ) + (d / "tests" / "test_cart.py").write_text( + """from cart import Cart +import pytest + + +def make_cart(): + c = Cart() + c.add("apple", 1.25, 4) + c.add("bread", 3.50) + c.add("milk", 2.15, 2) + return c + + +def test_empty_subtotal(): + assert Cart().subtotal() == 0 + + +def test_add_single(): + c = Cart() + c.add("apple", 1.25) + assert c.subtotal() == 1.25 + + +def test_add_qty(): + c = Cart() + c.add("apple", 1.25, 4) + assert c.subtotal() == 5.0 + + +def test_count(): + assert make_cart().count() == 7 + + +def test_subtotal_mixed(): + assert make_cart().subtotal() == pytest.approx(12.80) + + +def test_total_no_discount(): + assert make_cart().total() == pytest.approx(12.80) + + +def test_total_discount(): + assert make_cart().total(discount_pct=10) == pytest.approx(11.52) + + +def test_total_discount_rounds_once(): + c = Cart() + c.add("pen", 0.99, 3) + assert c.total(discount_pct=15) == pytest.approx(2.52, abs=0.005) + + +def test_qty_validation(): + c = Cart() + with pytest.raises(ValueError): + c.add("apple", 1.0, 0) + + +def test_add_many_lines(): + c = Cart() + for i in range(10): + c.add(f"item{i}", 1.0) + assert c.count() == 10 + + +def test_subtotal_after_many(): + c = Cart() + for i in range(10): + c.add(f"item{i}", 2.5) + assert c.subtotal() == 25.0 + + +def test_zero_discount_is_subtotal(): + c = make_cart() + assert c.total(0) == pytest.approx(c.subtotal()) +""" + ) + (d / "tests" / "test_textutil.py").write_text( + """from textutil import slugify, truncate_words + + +def test_slug_basic(): + assert slugify("Hello World") == "hello-world" + + +def test_slug_punctuation(): + assert slugify("a, b, c!") == "a-b-c" + + +def test_slug_leading_junk(): + assert slugify("--Already--Slugged--") == "already-slugged" + + +def test_slug_unicode_dropped(): + assert slugify("café au lait") == "café-au-lait" + + +def test_slug_empty(): + assert slugify("") == "" + + +def test_slug_numbers(): + assert slugify("v1.2.3 release") == "v1-2-3-release" + + +def test_truncate_short(): + assert truncate_words("one two", 5) == "one two" + + +def test_truncate_exact(): + assert truncate_words("one two three", 3) == "one two three" + + +def test_truncate_long(): + assert truncate_words("one two three four five", 3) == "one two three..." + + +def test_truncate_one(): + assert truncate_words("alpha beta", 1) == "alpha..." + + +def test_slug_spaces_collapse(): + assert slugify("a b") == "a-b" + + +def test_slug_mixed_case(): + assert slugify("MiXeD CaSe") == "mixed-case" +""" + ) + (d / "orders.py").write_text( + '''"""Order handling (sample project for the eco benchmark).""" + + +class OrderError(Exception): + pass + + +def _validate_qty(qty): + if qty < 1: + raise OrderError(f"qty must be >= 1, got {qty}") + return qty + + +def _unit_price(catalog, sku): + try: + return catalog[sku] + except KeyError: + raise OrderError(f"unknown sku: {sku}") from None + + +class Order: + def __init__(self, catalog): + self.catalog = catalog + self.lines = [] + + def add(self, sku, qty=1): + self.lines.append((sku, _validate_qty(qty))) + + def total(self): + return sum(_unit_price(self.catalog, sku) * qty for sku, qty in self.lines) + + def summary(self): + # BUG: counts order lines, not items. + return f"{len(self.lines)} items, total ${self.total():.2f}" +''' + ) + (d / "tests" / "test_orders.py").write_text( + """from orders import Order, OrderError +import pytest + +CATALOG = {"tea": 4.50, "mug": 11.00, "spoon": 1.25} + + +def make_order(): + o = Order(CATALOG) + o.add("tea", 2) + o.add("mug", 1) + return o + + +def test_total(): + assert make_order().total() == pytest.approx(20.0) + + +def test_add_validates_qty(): + with pytest.raises(OrderError): + make_order().add("tea", 0) + + +def test_unknown_sku_raises(): + o = Order(CATALOG) + o.add("gold-bar") + with pytest.raises(OrderError): + o.total() + + +def test_unknown_sku_message(): + o = Order(CATALOG) + o.add("gold-bar") + with pytest.raises(OrderError, match="unknown sku 'gold-bar'"): + o.total() + + +def test_summary_counts_items(): + o = Order(CATALOG) + o.add("tea", 2) + o.add("spoon", 3) + assert o.summary() == "5 items, total $12.75" + + +def test_summary_total_formatting(): + assert make_order().summary().endswith("$20.00") + + +def test_total_with_seasonal_catalog(): + seasonal = {"tea": 4.50, "mug": 11.00} + o = Order(seasonal) + o.add("tea") + o.add("cinnamon", 2) + assert o.total() == pytest.approx(8.10) + + +def test_empty_order(): + assert Order(CATALOG).total() == 0 + + +def test_multi_line_total(): + o = make_order() + o.add("spoon", 4) + assert o.total() == pytest.approx(25.0) + + +def test_lines_accumulate(): + o = make_order() + assert len(o.lines) == 2 +""" + ) + (d / "pytest.ini").write_text("[pytest]\npythonpath = .\ntestpaths = tests\n") + return d + + +def build_gosample(root: Path) -> Path: + """Go module, 3 packages, 2 with genuinely failing tests.""" + d = root / "gosample" + for pkg in ("mathx", "strfmt", "version"): + (d / pkg).mkdir(parents=True, exist_ok=True) + (d / "go.mod").write_text("module sample.dev/gosample\n\ngo 1.21\n") + (d / "mathx" / "mathx.go").write_text( + """package mathx + +func Clamp(v, lo, hi int) int { + if v < lo { + return lo + } + if v > hi { + return hi + } + return v +} + +// Mean returns the arithmetic mean. +// BUG: integer division happens before the float conversion. +func Mean(xs []int) float64 { + if len(xs) == 0 { + return 0 + } + sum := 0 + for _, x := range xs { + sum += x + } + return float64(sum / len(xs)) +} +""" + ) + (d / "mathx" / "median.go").write_text( + """package mathx + +import "sort" + +// Median returns the middle value. +// BUG: sorts the caller's slice in place AND mishandles even lengths. +func Median(xs []int) float64 { + if len(xs) == 0 { + return 0 + } + sort.Ints(xs) + return float64(xs[len(xs)/2]) +} +""" + ) + (d / "mathx" / "mathx_test.go").write_text( + """package mathx + +import "testing" + +func TestClamp(t *testing.T) { + cases := []struct{ v, lo, hi, want int }{ + {5, 0, 10, 5}, {-3, 0, 10, 0}, {42, 0, 10, 10}, + } + for _, c := range cases { + if got := Clamp(c.v, c.lo, c.hi); got != c.want { + t.Errorf("Clamp(%d,%d,%d) = %d, want %d", c.v, c.lo, c.hi, got, c.want) + } + } +} + +func TestMeanWhole(t *testing.T) { + if got := Mean([]int{2, 4, 6}); got != 4.0 { + t.Errorf("Mean = %v, want 4.0", got) + } +} + +func TestMeanFractional(t *testing.T) { + if got := Mean([]int{1, 2, 2}); got < 1.66 || got > 1.67 { + t.Errorf("Mean([1 2 2]) = %v, want ~1.667", got) + } +} + +func TestMedianTable(t *testing.T) { + cases := []struct { + name string + in []int + want float64 + }{ + {"odd sorted", []int{1, 3, 9}, 3}, + {"odd unsorted", []int{9, 1, 3}, 3}, + {"even mid", []int{1, 2, 3, 4}, 2.5}, + {"even unsorted", []int{4, 1, 3, 2}, 2.5}, + {"single", []int{7}, 7}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := Median(c.in); got != c.want { + t.Errorf("Median(%v) = %v, want %v", c.in, got, c.want) + } + }) + } +} + +func TestMedianDoesNotMutate(t *testing.T) { + in := []int{9, 1, 3} + Median(in) + if in[0] != 9 { + t.Errorf("Median mutated its input: %v", in) + } +} +""" + ) + (d / "strfmt" / "strfmt.go").write_text( + """package strfmt + +import "strings" + +// PadLeft pads s with spaces to exactly width runes. +// BUG: off by one when padding is needed. +func PadLeft(s string, width int) string { + if len(s) >= width { + return s + } + return strings.Repeat(" ", width-len(s)-1) + s +} + +func Ellipsis(s string, max int) string { + if len(s) <= max { + return s + } + return s[:max-3] + "..." +} +""" + ) + (d / "strfmt" / "strfmt_test.go").write_text( + """package strfmt + +import "testing" + +func TestPadLeft(t *testing.T) { + cases := []struct { + in string + width int + want string + }{ + {"7", 3, " 7"}, + {"42", 5, " 42"}, + {"abc", 3, "abc"}, + {"abcdef", 3, "abcdef"}, + {"", 2, " "}, + } + for _, c := range cases { + if got := PadLeft(c.in, c.width); got != c.want { + t.Errorf("PadLeft(%q, %d) = %q, want %q", c.in, c.width, got, c.want) + } + } +} + +func TestEllipsis(t *testing.T) { + if got := Ellipsis("hello world", 8); got != "hello..." { + t.Errorf("Ellipsis = %q", got) + } + if got := Ellipsis("hi", 8); got != "hi" { + t.Errorf("Ellipsis short = %q", got) + } +} +""" + ) + (d / "version" / "version.go").write_text( + "package version\n\nconst Current = \"1.4.2\"\n" + ) + (d / "version" / "version_test.go").write_text( + """package version + +import "testing" + +func TestCurrent(t *testing.T) { + if Current == "" { + t.Fatal("version must not be empty") + } +} +""" + ) + return d + + +_DURATION_BUGGY = """// Duration formatting (sample project for the eco benchmark). +// BUG: minutes are not zero-padded. +function formatDuration(totalSeconds) { + const h = Math.floor(totalSeconds / 3600); + const m = Math.floor((totalSeconds % 3600) / 60); + const s = totalSeconds % 60; + const pad = (n) => String(n).padStart(2, "0"); + if (h > 0) return `${h}h ${m}m ${pad(s)}s`; + return `${m}m ${pad(s)}s`; +} + +module.exports = { formatDuration }; +""" + +_DURATION_FIXED = _DURATION_BUGGY.replace( + "return `${h}h ${m}m ${pad(s)}s`", "return `${h}h ${pad(m)}m ${pad(s)}s`" +).replace("// BUG: minutes are not zero-padded.\n", "") + + +def build_jssample(root: Path) -> Path: + """Node project with a jest suite; 2 of 10 tests fail on a real bug.""" + d = root / "jssample" + d.mkdir(parents=True, exist_ok=True) + (d / "package.json").write_text( + json.dumps( + { + "name": "jssample", + "version": "1.0.0", + "license": "MIT", + "scripts": {"test": "jest"}, + }, + indent=2, + ) + ) + (d / "duration.js").write_text(_DURATION_BUGGY) + (d / "duration.test.js").write_text( + """const { formatDuration } = require("./duration"); + +test("seconds only", () => expect(formatDuration(7)).toBe("0m 07s")); +test("minutes and seconds", () => expect(formatDuration(125)).toBe("2m 05s")); +test("exact minute", () => expect(formatDuration(60)).toBe("1m 00s")); +test("hours pad minutes", () => expect(formatDuration(3723)).toBe("1h 02m 03s")); +test("hours zero minutes", () => expect(formatDuration(3605)).toBe("1h 00m 05s")); +test("zero", () => expect(formatDuration(0)).toBe("0m 00s")); +""" + ) + (d / "slug.js").write_text( + """function slugify(text) { + return text + .toLowerCase() + .trim() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, ""); +} + +module.exports = { slugify }; +""" + ) + (d / "slug.test.js").write_text( + """const { slugify } = require("./slug"); + +test("basic", () => expect(slugify("Hello World")).toBe("hello-world")); +test("punctuation", () => expect(slugify("a, b, c!")).toBe("a-b-c")); +test("edges", () => expect(slugify("--x--")).toBe("x")); +test("empty", () => expect(slugify("")).toBe("")); +""" + ) + return d + + +# ── capture sweep ──────────────────────────────────────────────────────────── + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--workdir", required=True, help="scratch dir for sample projects") + ap.add_argument("--corpus", default=str(CORPUS_DIR), help="output corpus dir") + ap.add_argument("--repo", default=str(REPO_ROOT), help="repository to benchmark against") + ap.add_argument("--python", default=None, help="python with pytest (default: repo venv)") + args = ap.parse_args() + + work = Path(args.workdir).resolve() + repo = Path(args.repo).resolve() + if work.exists(): + if any(work.iterdir()) and not (work / WORKDIR_MARKER).exists(): + raise SystemExit( + f"refusing to delete pre-existing non-empty {work} — it was not " + f"created by this script (no {WORKDIR_MARKER} marker); pass a " + "fresh --workdir path" + ) + shutil.rmtree(work) + work.mkdir(parents=True) + (work / WORKDIR_MARKER).write_text( + "scratch dir created by eval/eco/capture_corpus.py — safe to delete\n" + ) + cap = Capture(Path(args.corpus).resolve()) + py = _pytest_python(args.python) + + # 1. pytest — sample project with real failures, plain + verbose + green. + print("== pytest (sample project) ==") + pysample = build_pysample(work) + cap.run( + "pytest-fail", "pytest (failing run)", "pytest", + executed=f"{py} -m pytest", cwd=pysample, category="test-runner", + ) + cap.run( + "pytest-fail-verbose", "pytest -v (failing run)", "pytest -v", + executed=f"{py} -m pytest -v", cwd=pysample, category="test-runner", + ) + cap.run( + "pytest-green-verbose", "pytest -v (green run)", + "pytest -v tests/test_cart.py", + executed=f"{py} -m pytest -v tests/test_cart.py", + cwd=pysample, category="test-runner", + ) + + # 2. pytest — a green slice of this repository's real suite. + print("== pytest (this repo, green slice) ==") + cap.run( + "pytest-green-repo", + "pytest (repo suite slice, all pass)", + "pytest tests/test_eco.py tests/test_bash_eco_integration.py", + executed=f"{py} -m pytest tests/test_eco.py tests/test_bash_eco_integration.py", + cwd=repo, category="test-runner", timeout=600, + ) + + # 3. go test — sample module with real failures. + if shutil.which("go"): + print("== go test ==") + gosample = build_gosample(work) + cap.run( + "go-test-fail", "go test ./... (failing run)", "go test ./...", + cwd=gosample, category="test-runner", timeout=300, + ) + cap.run( + "go-test-fail-verbose", "go test -v ./... (failing run)", + "go test -v ./...", cwd=gosample, category="test-runner", timeout=300, + ) + else: + print("== go not installed; skipping go test ==") + + # 4. jest — npm install (ceremony corpus) + failing + green runs. + if shutil.which("npm"): + print("== jest ==") + jssample = build_jssample(work) + cap.run( + "npm-install", "npm install jest", "npm install --no-fund --no-audit jest", + cwd=jssample, category="package-manager", timeout=600, + ) + cap.run( + "jest-fail", "npx jest (failing run)", "npx jest", + cwd=jssample, category="test-runner", timeout=300, + ) + (jssample / "duration.js").write_text(_DURATION_FIXED) + cap.run( + "jest-green", "npx jest (green run)", "npx jest", + cwd=jssample, category="test-runner", timeout=300, + ) + else: + print("== npm not installed; skipping jest ==") + + # 5. pip install into a fresh venv (Collecting/Downloading ceremony). + print("== pip install ==") + pipvenv = work / "pipvenv" + try: + subprocess.run( + [sys.executable, "-m", "venv", str(pipvenv)], + check=True, capture_output=True, timeout=300, + ) + except (OSError, subprocess.SubprocessError) as exc: + print(f" [SKIP] pip-install + ruff-check: venv creation failed ({exc})") + pipvenv = None + if pipvenv is not None: + pip = pipvenv / "bin" / "pip" + cap.run( + "pip-install", "pip install flask", "pip install --no-cache-dir flask", + executed=f"{pip} install --no-cache-dir flask", + cwd=work, category="package-manager", timeout=600, + ) + + # 6. ruff check — real lint errors, no eco filter (honest passthrough). + try: + subprocess.run( + [str(pip), "install", "--quiet", "ruff"], + capture_output=True, timeout=600, + ) + except (OSError, subprocess.SubprocessError) as exc: + print(f" [SKIP] ruff install: {exc}") + ruff = pipvenv / "bin" / "ruff" + if ruff.exists(): + cap.run( + "ruff-check", "ruff check (findings)", "ruff check .", + executed=f"{ruff} check .", cwd=pysample, category="lint", + ) + else: + print(" [SKIP] ruff-check: ruff not installed") + + # 7. git — clone with progress, dirty status, commit, push to a local bare. + print("== git ==") + clone = work / "clawcodex-clone" + cap.run( + "git-clone", "git clone --progress", + f"git clone --no-local --progress {repo} {clone}", + cwd=work, category="git", timeout=600, + ) + if clone.exists(): + # Clean-tree status is captured in the pristine clone (the benchmark + # repo itself may be mid-change while this script runs). + cap.run( + "git-status-clean", "git status (clean tree)", "git status", + cwd=clone, category="passthrough", + ) + for rel in ("README.md", "pyproject.toml", "src/eco/state.py"): + p = clone / rel + if p.exists(): + with p.open("a") as fh: + fh.write("\n# local experiment\n") + (clone / "notes.txt").write_text("scratch notes\n") + (clone / "scratch").mkdir(exist_ok=True) + (clone / "scratch" / "probe.py").write_text("print('probe')\n") + cap.run( + "git-status-dirty", "git status (3 modified, 2 untracked)", "git status", + cwd=clone, category="git", + ) + bare = work / "push-target.git" + subprocess.run( + ["git", "init", "--bare", str(bare)], check=True, capture_output=True + ) + subprocess.run( + ["git", "remote", "add", "target", str(bare)], + cwd=clone, check=True, capture_output=True, + ) + subprocess.run(["git", "add", "-A"], cwd=clone, check=True, capture_output=True) + cap.run( + "git-commit", "git commit", 'git commit -m "wip: local experiment"', + cwd=clone, category="git", + ) + branch = subprocess.run( + ["git", "branch", "--show-current"], cwd=clone, + capture_output=True, text=True, + ).stdout.strip() or "main" + cap.run( + "git-push", "git push --progress", + f"git push --progress target {branch}", + cwd=clone, category="git", timeout=300, + ) + + # 8. repo-scale reads: history, diffs, listings, big files, grep sweeps. + print("== repo-scale reads ==") + cap.run("git-log-full", "git log -n 300 (full format)", "git log -n 300", + cwd=repo, category="listing") + cap.run("git-log-oneline", "git log --oneline (full history)", + "git log --oneline", cwd=repo, category="listing") + diff_range = "v1.0.0..v1.1.0" + have_tags = subprocess.run( + ["git", "rev-parse", "v1.0.0", "v1.1.0"], cwd=repo, capture_output=True + ).returncode == 0 + if not have_tags: + diff_range = "HEAD~30..HEAD" + cap.run("git-diff-large", f"git diff {diff_range} -- src", + f"git diff {diff_range} -- src", cwd=repo, category="listing") + cap.run("ls-R", "ls -R src (whole tree)", "ls -R src", cwd=repo, category="listing") + cap.run("find-py", "find . -name '*.py'", + "find . -type f -name '*.py' -not -path './.git/*'", + cwd=repo, category="listing") + cap.run("cat-large", "cat (900-line source file)", + "cat src/tool_system/tools/bash/bash_tool.py", cwd=repo, category="listing") + cap.run("grep-defs", "grep -rn 'def ' (subsystem sweep)", + "grep -rn 'def ' src/tool_system/", cwd=repo, category="listing") + cap.run("grep-defs-wide", "grep -rn 'def ' src/ (repo-wide)", + "grep -rn 'def ' src/", cwd=repo, category="listing") + + # 9. log-shaped output — macOS unified log (real timestamped log lines). + if platform.system() == "Darwin": + print("== system log ==") + cap.run( + "log-dedup", "log show --last 90s (system log)", + "log show --last 90s --style syslog", + cwd=work, category="logs", timeout=300, + ) + + # 10. small outputs — must pass through byte-identical (never-worse rows). + print("== small passthrough rows ==") + cap.run("wc-readme", "wc -l README.md", "wc -l README.md", + cwd=repo, category="passthrough") + docker_up = False + if shutil.which("docker"): + try: + docker_up = ( + subprocess.run( + ["docker", "info"], capture_output=True, timeout=20 + ).returncode + == 0 + ) + except (OSError, subprocess.TimeoutExpired): + docker_up = False + if docker_up: + cap.run("docker-ps", "docker ps", "docker ps", cwd=work, category="passthrough") + else: + print(" [SKIP] docker-ps: no running docker daemon") + + print(f"\ncaptured {len(cap.ok)} items → {cap.corpus_dir}") + if cap.failed: + print(f"failed: {', '.join(cap.failed)}") + # Partial failures exit non-zero so a benchmark run can't silently lose rows. + return 0 if (cap.ok and not cap.failed) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/eco/measure.py b/eval/eco/measure.py new file mode 100644 index 000000000..019688d8f --- /dev/null +++ b/eval/eco/measure.py @@ -0,0 +1,340 @@ +#!/usr/bin/env python3 +"""Measure /eco token savings over a captured corpus (see capture_corpus.py). + +Each corpus item is replayed through the *production* pipeline, byte-for-byte +what the Bash tool does when ``/eco`` is on: + + baseline = _assemble_bash_body(truncate_output(stdout), truncate_output(stderr)) + outcome = compress_bash_output(command, exit_code, full_text, baseline, tee_dir) + +and both renderings get the mapper's ``returnCodeInterpretation`` suffix, so +the counted text is exactly the wire content with ``/eco`` off vs on. Tokens +are counted with tiktoken ``cl100k_base`` — real tokenizer counts, not the +chars/4 estimate RTK reports (chars/4 is also computed, into the JSON, for +comparability with RTK's methodology). + +Determinism: tee filenames embed ``time_ns``/``pid``; both are pinned here and +the tee dir is fixed (``corpus/.tee``), so on an unchanged corpus this script +is a pure function — reruns must not churn the committed ``results/``. The +recovery-hint line is charged to eco at the bench path's length; for the +committed run that tokenizes no shorter than a typical production +``~/.clawcodex/...`` hint, so its published savings are, if anything, slightly +understated. + +Outputs (default ``eval/eco/results/``): + results.md — per-operation table + corpus totals + RTK session projection + results.json — full numbers, per item + examples/ — before/after renderings for a few showcase items + +Run from the repo root with the project venv: + .venv/bin/python eval/eco/measure.py [--corpus DIR] [--results DIR] +""" + +from __future__ import annotations + +import argparse +import itertools +import json +import shutil +import sys +import types +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parents[1] +sys.path.insert(0, str(REPO_ROOT)) + +import src.eco.tee as _tee # noqa: E402 +from src.eco.engine import compress_bash_output # noqa: E402 +from src.eco.guard import estimate_tokens as chars4_tokens # noqa: E402 +from src.tool_system.tools.bash.bash_tool import _assemble_bash_body # noqa: E402 +from src.tool_system.tools.bash.command_semantics import ( # noqa: E402 + interpret_command_result, +) +from src.tool_system.tools.bash.utils import truncate_output # noqa: E402 + +try: + import tiktoken +except ImportError: # pragma: no cover — measurement requires the real tokenizer + sys.exit("measure.py needs tiktoken (the README numbers are real token counts): pip install tiktoken") + +_ENC = tiktoken.get_encoding("cl100k_base") + +CATEGORY_ORDER = ( + "test-runner", "package-manager", "git", "listing", "logs", "lint", "passthrough", +) + +EXAMPLE_LABELS = ("pytest-fail", "git-status-dirty", "npm-install") + + +def _pin_tee_determinism() -> None: + """Tee filenames are ``{time_ns}_{pid}_{counter}_{slug}.log``; pin the + volatile parts so identical corpora produce identical recovery hints + (reruns must not churn the committed results/examples).""" + _tee.time = types.SimpleNamespace(time_ns=lambda: 1707) + + real_os = _tee.os + + class _PinnedOS: + def __getattr__(self, name): + return getattr(real_os, name) + + @staticmethod + def getpid() -> int: + return 0 + + _tee.os = _PinnedOS() + _tee._counter = itertools.count() + + +def tok(text: str) -> int: + return len(_ENC.encode(text, disallowed_special=())) + + +def measure_item(item: dict, tee_dir: Path) -> dict: + baseline = _assemble_bash_body( + truncate_output(item["stdout"]), truncate_output(item["stderr"]) + ) + full_text = _assemble_bash_body(item["stdout"], item["stderr"]) + outcome = compress_bash_output( + command=item["command"], + exit_code=item["exit_code"], + full_text=full_text, + baseline=baseline, + tee_dir=tee_dir, + ) + final = outcome.content if outcome is not None else baseline + + # The mapper appends returnCodeInterpretation after the body on BOTH + # paths (bash_tool._bash_map_result_to_api); include it so the counted + # text is exactly the wire content. + interp = interpret_command_result( + item["command"], item["exit_code"], item["stdout"], item["stderr"] + ).message + baseline_wire = f"{baseline}\n{interp}" if interp else baseline + final_wire = f"{final}\n{interp}" if interp else final + + b_tok, f_tok = tok(baseline_wire), tok(final_wire) + saved = b_tok - f_tok + return { + "label": item["label"], + "op": item["op"], + "category": item.get("category", ""), + "command": item["command"], + "exit_code": item["exit_code"], + "filter": outcome.filter_name if outcome else None, + "baseline_chars": len(baseline_wire), + "eco_chars": len(final_wire), + "baseline_tokens": b_tok, + "eco_tokens": f_tok, + "saved_tokens": saved, + "saved_pct": round(100.0 * saved / b_tok, 1) if b_tok else 0.0, + "baseline_tokens_chars4": chars4_tokens(baseline_wire), + "eco_tokens_chars4": chars4_tokens(final_wire), + "baseline_text": baseline_wire, + "eco_text": final_wire, + } + + +# ── RTK 30-minute-session projection ───────────────────────────────────────── +# RTK's README models a 30-minute Claude Code session as a frequency-weighted +# mix of operations and claims -80% overall (~118,000 → ~23,900 tokens), +# estimated, per their caveat, on "medium-sized TypeScript/Rust projects". +# We recompute the SAME session model with our *measured* savings ratios. +# Where RTK's assumed per-call size sits below eco's thresholds (a ~200-token +# `ls`, a ~2,000-token `cat` — under the 400-line head-cap), eco passes the +# output through untouched, so those rows honestly contribute 0%. +# +# rows: (operation, freq, rtk_standard_tokens, rtk_tokens, [(label|None, share)]) +RTK_SESSION = ( + ("`ls` / `tree`", 10, 2000, 400, ((None, 1.0),)), + ("`cat` / read", 20, 40000, 12000, ((None, 1.0),)), + ("`grep` / `rg`", 8, 16000, 3200, ((None, 1.0),)), + ("`git status`", 10, 3000, 600, (("git-status-dirty", 1.0),)), + ("`git diff`", 5, 10000, 2500, ((None, 1.0),)), + ("`git log`", 5, 2500, 500, ((None, 1.0),)), + # 8 calls ≈ add + commit + push cycles; only push output compresses. + ("`git add/commit/push`", 8, 1600, 120, (("git-push", 0.375), (None, 0.625))), + ("`cargo test` / `npm test`", 5, 25000, 2500, (("jest-fail", 1.0),)), + ("`ruff check`", 3, 3000, 600, (("ruff-check", 1.0),)), + ("`pytest`", 4, 8000, 800, (("pytest-fail", 1.0),)), + ("`go test`", 3, 6000, 600, (("go-test-fail", 1.0),)), + ("`docker ps`", 3, 900, 180, (("docker-ps", 1.0),)), +) + +# RTK's README states the totals as ~118,000 → ~23,900 (-80%); its own rows +# sum to 24,000. Quote their claimed total for the comparison line. +RTK_CLAIMED_TOTAL = 23_900 + + +def project_session(by_label: dict[str, dict]) -> tuple[list[dict], int, int]: + rows = [] + std_total = ours_total = 0 + for op, freq, std, rtk, mix in RTK_SESSION: + ours = 0.0 + for label, share in mix: + m = by_label.get(label) if label else None + ratio = (m["saved_pct"] / 100.0) if m else 0.0 + ours += std * share * (1.0 - ratio) + rows.append( + { + "op": op, "freq": freq, "standard": std, "rtk": rtk, + "eco": int(round(ours)), + # Floor to match fmt_pct — display never overstates savings. + "eco_pct": float(int(100.0 * (1 - ours / std))) if std else 0.0, + } + ) + std_total += std + ours_total += int(round(ours)) + return rows, std_total, ours_total + + +def fmt_pct(saved_pct: float) -> str: + """Floor, not round: a 99.7% saving displays as -99%, never a false -100%.""" + floored = int(saved_pct) + return f"-{floored}%" if floored >= 1 else "0%" + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--corpus", default=str(HERE / "corpus"), help="captured corpus dir") + ap.add_argument("--results", default=str(HERE / "results"), help="output dir") + args = ap.parse_args() + + corpus_dir = Path(args.corpus).resolve() + results_dir = Path(args.results).resolve() + examples_dir = results_dir / "examples" + results_dir.mkdir(parents=True, exist_ok=True) + examples_dir.mkdir(exist_ok=True) + + items = [] + for path in sorted(corpus_dir.glob("*.json")): + items.append(json.loads(path.read_text())) + if not items: + sys.exit(f"no corpus at {corpus_dir} — run capture_corpus.py first") + + # Fixed, gitignored tee dir; cleared so counter-based names are stable. + tee_dir = corpus_dir / ".tee" + if tee_dir.exists(): + shutil.rmtree(tee_dir) + _pin_tee_determinism() + + measured = [measure_item(it, tee_dir) for it in items] + measured.sort( + key=lambda m: ( + CATEGORY_ORDER.index(m["category"]) + if m["category"] in CATEGORY_ORDER + else len(CATEGORY_ORDER), + m["label"], + ) + ) + by_label = {m["label"]: m for m in measured} + + # Never-worse invariant: eco must not emit more (estimated) tokens than + # the baseline it replaced. The guard operates on chars/4. + violations = [ + m["label"] + for m in measured + if m["eco_tokens_chars4"] > m["baseline_tokens_chars4"] + ] + if violations: + sys.exit(f"never-worse violated (bug!): {violations}") + + # ── per-operation table ── + lines = [ + "| Operation | Filter | Raw tokens | /eco tokens | Saved |", + "|---|---|---:|---:|---:|", + ] + for m in measured: + lines.append( + f"| {m['op']} | {m['filter'] or '— (passthrough)'} " + f"| {m['baseline_tokens']:,} | {m['eco_tokens']:,} " + f"| **{fmt_pct(m['saved_pct'])}** |" + ) + b_sum = sum(m["baseline_tokens"] for m in measured) + e_sum = sum(m["eco_tokens"] for m in measured) + total_pct = 100 * (b_sum - e_sum) / b_sum if b_sum else 0.0 + lines.append( + f"| **Corpus total** | | **{b_sum:,}** | **{e_sum:,}** " + f"| **-{int(total_pct)}%** |" + ) + hits = [m for m in measured if m["filter"]] + hb = sum(m["baseline_tokens"] for m in hits) + he = sum(m["eco_tokens"] for m in hits) + hits_pct = 100 * (hb - he) / hb if hb else 0.0 + + # ── RTK session projection ── + srows, s_std, s_eco = project_session(by_label) + slines = [ + "| Operation | Freq | Standard | rtk (claimed) | clawcodex /eco (measured) |", + "|---|---:|---:|---:|---:|", + ] + for r in srows: + slines.append( + f"| {r['op']} | {r['freq']}x | {r['standard']:,} | {r['rtk']:,} " + f"| {r['eco']:,} ({fmt_pct(r['eco_pct'])}) |" + ) + slines.append( + f"| **Total** | | **~{s_std:,}** | **~{RTK_CLAIMED_TOTAL:,} (-80%)** " + f"| **~{s_eco:,} (-{int(100 * (s_std - s_eco) / s_std)}%)** |" + ) + + md = "\n".join( + [ + "# /eco benchmark results", + "", + "Generated by `eval/eco/measure.py` over the corpus captured by", + "`eval/eco/capture_corpus.py`. Tokens are tiktoken `cl100k_base` counts", + "of the exact wire content (baseline vs eco rendering, including the", + "recovery-hint line and `returnCodeInterpretation` suffix).", + "", + "## Per-operation (measured on real outputs)", + "", + *lines, + "", + f"Filter-hit subset: {hb:,} → {he:,} tokens " + f"(**-{int(hits_pct)}%**) across {len(hits)} operations; " + f"{len(measured) - len(hits)} operations passed through unchanged (0%).", + "", + "## RTK 30-minute-session model, recomputed with measured ratios", + "", + *slines, + "", + ] + ) + (results_dir / "results.md").write_text(md) + (results_dir / "results.json").write_text( + json.dumps( + { + "tokenizer": "cl100k_base", + "items": [ + {k: v for k, v in m.items() if not k.endswith("_text")} + for m in measured + ], + "corpus_total": {"baseline": b_sum, "eco": e_sum}, + "filter_hits_total": {"baseline": hb, "eco": he}, + "rtk_session_projection": { + "rows": srows, + "standard": s_std, + "rtk_claimed": RTK_CLAIMED_TOTAL, + "eco": s_eco, + }, + }, + indent=1, + ) + ) + + for label in EXAMPLE_LABELS: + m = by_label.get(label) + if m is None: + continue + (examples_dir / f"{label}.before.txt").write_text(m["baseline_text"]) + (examples_dir / f"{label}.after.txt").write_text(m["eco_text"]) + + print(md) + print(f"\nwrote {results_dir}/results.md, results.json, examples/") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/eval/eco/results/examples/git-status-dirty.after.txt b/eval/eco/results/examples/git-status-dirty.after.txt new file mode 100644 index 000000000..4523bdac8 --- /dev/null +++ b/eval/eco/results/examples/git-status-dirty.after.txt @@ -0,0 +1,13 @@ +On branch worktree-RTK-token-compression +Your branch is up to date with 'origin/worktree-RTK-token-compression'. + +Changes not staged for commit: + modified: README.md + modified: pyproject.toml + modified: src/eco/state.py + +Untracked files: + notes.txt + scratch/ + +no changes added to commit (use "git add" and/or "git commit -a") \ No newline at end of file diff --git a/eval/eco/results/examples/git-status-dirty.before.txt b/eval/eco/results/examples/git-status-dirty.before.txt new file mode 100644 index 000000000..2dc95ba60 --- /dev/null +++ b/eval/eco/results/examples/git-status-dirty.before.txt @@ -0,0 +1,16 @@ +On branch worktree-RTK-token-compression +Your branch is up to date with 'origin/worktree-RTK-token-compression'. + +Changes not staged for commit: + (use "git add ..." to update what will be committed) + (use "git restore ..." to discard changes in working directory) + modified: README.md + modified: pyproject.toml + modified: src/eco/state.py + +Untracked files: + (use "git add ..." to include in what will be committed) + notes.txt + scratch/ + +no changes added to commit (use "git add" and/or "git commit -a") \ No newline at end of file diff --git a/eval/eco/results/examples/npm-install.after.txt b/eval/eco/results/examples/npm-install.after.txt new file mode 100644 index 000000000..7447e9444 --- /dev/null +++ b/eval/eco/results/examples/npm-install.after.txt @@ -0,0 +1 @@ +added 294 packages in 4s \ No newline at end of file diff --git a/eval/eco/results/examples/npm-install.before.txt b/eval/eco/results/examples/npm-install.before.txt new file mode 100644 index 000000000..11e9aa51c --- /dev/null +++ b/eval/eco/results/examples/npm-install.before.txt @@ -0,0 +1,4 @@ +added 294 packages in 4s +npm warn deprecated inflight@1.0.6: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. +npm warn deprecated glob@7.2.3: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me +npm warn deprecated glob@10.5.0: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me \ No newline at end of file diff --git a/eval/eco/results/examples/pytest-fail.after.txt b/eval/eco/results/examples/pytest-fail.after.txt new file mode 100644 index 000000000..2569ffbad --- /dev/null +++ b/eval/eco/results/examples/pytest-fail.after.txt @@ -0,0 +1,38 @@ +Pytest: 5 failed, 29 passed in 0.04s + +1. [FAIL] test_unknown_sku_message + with pytest.raises(OrderError, match="unknown sku 'gold-bar'"): + > o.total() + tests/test_orders.py:34: + orders.py:30: in total + orders.py:30: in + +2. [FAIL] test_summary_counts_items + > assert o.summary() == "5 items, total $12.75" + E AssertionError: assert '2 items, total $12.75' == '5 items, total $12.75' + E + E - 5 items, total $12.75 + E ? ^ + +3. [FAIL] test_total_with_seasonal_catalog + > assert o.total() == pytest.approx(8.10) + tests/test_orders.py:53: + orders.py:30: in total + orders.py:30: in + except KeyError: + +4. [FAIL] test_truncate_long + > assert truncate_words("one two three four five", 3) == "one two three..." + E AssertionError: assert 'one two three four...' == 'one two three...' + E + E - one two three... + E + one two three four... + +5. [FAIL] test_truncate_one + > assert truncate_words("alpha beta", 1) == "alpha..." + E AssertionError: assert 'alpha beta...' == 'alpha...' + E + E - alpha... + E + alpha beta... +[full output: ~/workspace/clawcodex/.claude/worktrees/RTK-token-compression/eval/eco/corpus/.tee/1707_0_11_pytest.log] +Command failed with exit code 1 \ No newline at end of file diff --git a/eval/eco/results/examples/pytest-fail.before.txt b/eval/eco/results/examples/pytest-fail.before.txt new file mode 100644 index 000000000..6c50d1e70 --- /dev/null +++ b/eval/eco/results/examples/pytest-fail.before.txt @@ -0,0 +1,129 @@ +============================= test session starts ============================== +platform darwin -- Python 3.11.14, pytest-9.0.2, pluggy-1.6.0 +rootdir: /private/tmp/eco-bench/pysample +configfile: pytest.ini +testpaths: tests +plugins: cov-7.1.0, asyncio-1.3.0, anyio-4.13.0 +asyncio: mode=Mode.STRICT, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function +collected 34 items + +tests/test_cart.py ............ [ 35%] +tests/test_orders.py ...FF.F... [ 64%] +tests/test_textutil.py ........FF.. [100%] + +=================================== FAILURES =================================== +___________________________ test_unknown_sku_message ___________________________ + + def test_unknown_sku_message(): + o = Order(CATALOG) + o.add("gold-bar") + with pytest.raises(OrderError, match="unknown sku 'gold-bar'"): +> o.total() + +tests/test_orders.py:34: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +orders.py:30: in total + return sum(_unit_price(self.catalog, sku) * qty for sku, qty in self.lines) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +orders.py:30: in + return sum(_unit_price(self.catalog, sku) * qty for sku, qty in self.lines) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +catalog = {'mug': 11.0, 'spoon': 1.25, 'tea': 4.5}, sku = 'gold-bar' + + def _unit_price(catalog, sku): + try: + return catalog[sku] + except KeyError: +> raise OrderError(f"unknown sku: {sku}") from None +E orders.OrderError: unknown sku: gold-bar + +orders.py:18: OrderError + +During handling of the above exception, another exception occurred: + + def test_unknown_sku_message(): + o = Order(CATALOG) + o.add("gold-bar") +> with pytest.raises(OrderError, match="unknown sku 'gold-bar'"): +E AssertionError: Regex pattern did not match. +E Expected regex: "unknown sku 'gold-bar'" +E Actual message: 'unknown sku: gold-bar' + +tests/test_orders.py:33: AssertionError +__________________________ test_summary_counts_items ___________________________ + + def test_summary_counts_items(): + o = Order(CATALOG) + o.add("tea", 2) + o.add("spoon", 3) +> assert o.summary() == "5 items, total $12.75" +E AssertionError: assert '2 items, total $12.75' == '5 items, total $12.75' +E +E - 5 items, total $12.75 +E ? ^ +E + 2 items, total $12.75 +E ? ^ + +tests/test_orders.py:41: AssertionError +_______________________ test_total_with_seasonal_catalog _______________________ + + def test_total_with_seasonal_catalog(): + seasonal = {"tea": 4.50, "mug": 11.00} + o = Order(seasonal) + o.add("tea") + o.add("cinnamon", 2) +> assert o.total() == pytest.approx(8.10) + ^^^^^^^^^ + +tests/test_orders.py:53: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ +orders.py:30: in total + return sum(_unit_price(self.catalog, sku) * qty for sku, qty in self.lines) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +orders.py:30: in + return sum(_unit_price(self.catalog, sku) * qty for sku, qty in self.lines) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +catalog = {'mug': 11.0, 'tea': 4.5}, sku = 'cinnamon' + + def _unit_price(catalog, sku): + try: + return catalog[sku] + except KeyError: +> raise OrderError(f"unknown sku: {sku}") from None +E orders.OrderError: unknown sku: cinnamon + +orders.py:18: OrderError +______________________________ test_truncate_long ______________________________ + + def test_truncate_long(): +> assert truncate_words("one two three four five", 3) == "one two three..." +E AssertionError: assert 'one two three four...' == 'one two three...' +E +E - one two three... +E + one two three four... +E ? +++++ + +tests/test_textutil.py:37: AssertionError +______________________________ test_truncate_one _______________________________ + + def test_truncate_one(): +> assert truncate_words("alpha beta", 1) == "alpha..." +E AssertionError: assert 'alpha beta...' == 'alpha...' +E +E - alpha... +E + alpha beta... +E ? +++++ + +tests/test_textutil.py:41: AssertionError +=========================== short test summary info ============================ +FAILED tests/test_orders.py::test_unknown_sku_message - AssertionError: Regex... +FAILED tests/test_orders.py::test_summary_counts_items - AssertionError: asse... +FAILED tests/test_orders.py::test_total_with_seasonal_catalog - orders.OrderE... +FAILED tests/test_textutil.py::test_truncate_long - AssertionError: assert 'o... +FAILED tests/test_textutil.py::test_truncate_one - AssertionError: assert 'al... +========================= 5 failed, 29 passed in 0.04s ========================= +Command failed with exit code 1 \ No newline at end of file diff --git a/eval/eco/results/results.json b/eval/eco/results/results.json new file mode 100644 index 000000000..fb8bc4175 --- /dev/null +++ b/eval/eco/results/results.json @@ -0,0 +1,548 @@ +{ + "tokenizer": "cl100k_base", + "items": [ + { + "label": "go-test-fail", + "op": "go test ./... (failing run)", + "category": "test-runner", + "command": "go test ./...", + "exit_code": 1, + "filter": null, + "baseline_chars": 806, + "eco_chars": 806, + "baseline_tokens": 322, + "eco_tokens": 322, + "saved_tokens": 0, + "saved_pct": 0.0, + "baseline_tokens_chars4": 202, + "eco_tokens_chars4": 202 + }, + { + "label": "go-test-fail-verbose", + "op": "go test -v ./... (failing run)", + "category": "test-runner", + "command": "go test -v ./...", + "exit_code": 1, + "filter": "go-test", + "baseline_chars": 1453, + "eco_chars": 647, + "baseline_tokens": 527, + "eco_tokens": 227, + "saved_tokens": 300, + "saved_pct": 56.9, + "baseline_tokens_chars4": 364, + "eco_tokens_chars4": 162 + }, + { + "label": "jest-fail", + "op": "npx jest (failing run)", + "category": "test-runner", + "command": "npx jest", + "exit_code": 1, + "filter": "jest", + "baseline_chars": 1500, + "eco_chars": 517, + "baseline_tokens": 444, + "eco_tokens": 175, + "saved_tokens": 269, + "saved_pct": 60.6, + "baseline_tokens_chars4": 375, + "eco_tokens_chars4": 130 + }, + { + "label": "jest-green", + "op": "npx jest (green run)", + "category": "test-runner", + "command": "npx jest", + "exit_code": 0, + "filter": null, + "baseline_chars": 141, + "eco_chars": 141, + "baseline_tokens": 50, + "eco_tokens": 50, + "saved_tokens": 0, + "saved_pct": 0.0, + "baseline_tokens_chars4": 36, + "eco_tokens_chars4": 36 + }, + { + "label": "pytest-fail", + "op": "pytest (failing run)", + "category": "test-runner", + "command": "pytest", + "exit_code": 1, + "filter": "pytest", + "baseline_chars": 5274, + "eco_chars": 1320, + "baseline_tokens": 1347, + "eco_tokens": 390, + "saved_tokens": 957, + "saved_pct": 71.0, + "baseline_tokens_chars4": 1319, + "eco_tokens_chars4": 330 + }, + { + "label": "pytest-fail-verbose", + "op": "pytest -v (failing run)", + "category": "test-runner", + "command": "pytest -v", + "exit_code": 1, + "filter": "pytest", + "baseline_chars": 7849, + "eco_chars": 1323, + "baseline_tokens": 1925, + "eco_tokens": 392, + "saved_tokens": 1533, + "saved_pct": 79.6, + "baseline_tokens_chars4": 1963, + "eco_tokens_chars4": 331 + }, + { + "label": "pytest-green-repo", + "op": "pytest (repo suite slice, all pass)", + "category": "test-runner", + "command": "pytest tests/test_eco.py tests/test_bash_eco_integration.py", + "exit_code": 0, + "filter": "pytest", + "baseline_chars": 684, + "eco_chars": 179, + "baseline_tokens": 170, + "eco_tokens": 63, + "saved_tokens": 107, + "saved_pct": 62.9, + "baseline_tokens_chars4": 171, + "eco_tokens_chars4": 45 + }, + { + "label": "pytest-green-verbose", + "op": "pytest -v (green run)", + "category": "test-runner", + "command": "pytest -v tests/test_cart.py", + "exit_code": 0, + "filter": "pytest", + "baseline_chars": 1533, + "eco_chars": 167, + "baseline_tokens": 359, + "eco_tokens": 60, + "saved_tokens": 299, + "saved_pct": 83.3, + "baseline_tokens_chars4": 384, + "eco_tokens_chars4": 42 + }, + { + "label": "npm-install", + "op": "npm install jest", + "category": "package-manager", + "command": "npm install --no-fund --no-audit jest", + "exit_code": 0, + "filter": "noise-strip", + "baseline_chars": 820, + "eco_chars": 24, + "baseline_tokens": 188, + "eco_tokens": 8, + "saved_tokens": 180, + "saved_pct": 95.7, + "baseline_tokens_chars4": 205, + "eco_tokens_chars4": 6 + }, + { + "label": "pip-install", + "op": "pip install flask", + "category": "package-manager", + "command": "pip install --no-cache-dir flask", + "exit_code": 0, + "filter": "noise-strip", + "baseline_chars": 1324, + "eco_chars": 222, + "baseline_tokens": 514, + "eco_tokens": 85, + "saved_tokens": 429, + "saved_pct": 83.5, + "baseline_tokens_chars4": 331, + "eco_tokens_chars4": 56 + }, + { + "label": "git-clone", + "op": "git clone --progress", + "category": "git", + "command": "git clone --no-local --progress /Users/ericlee2/workspace/clawcodex/.claude/worktrees/RTK-token-compression /private/tmp/eco-bench/clawcodex-clone", + "exit_code": 0, + "filter": "noise-strip", + "baseline_chars": 18401, + "eco_chars": 56, + "baseline_tokens": 6868, + "eco_tokens": 18, + "saved_tokens": 6850, + "saved_pct": 99.7, + "baseline_tokens_chars4": 4601, + "eco_tokens_chars4": 14 + }, + { + "label": "git-commit", + "op": "git commit", + "category": "git", + "command": "git commit -m \"wip: local experiment\"", + "exit_code": 0, + "filter": null, + "baseline_chars": 163, + "eco_chars": 163, + "baseline_tokens": 49, + "eco_tokens": 49, + "saved_tokens": 0, + "saved_pct": 0.0, + "baseline_tokens_chars4": 41, + "eco_tokens_chars4": 41 + }, + { + "label": "git-push", + "op": "git push --progress", + "category": "git", + "command": "git push --progress target worktree-RTK-token-compression", + "exit_code": 0, + "filter": "noise-strip", + "baseline_chars": 16363, + "eco_chars": 242, + "baseline_tokens": 6458, + "eco_tokens": 75, + "saved_tokens": 6383, + "saved_pct": 98.8, + "baseline_tokens_chars4": 4091, + "eco_tokens_chars4": 61 + }, + { + "label": "git-status-dirty", + "op": "git status (3 modified, 2 untracked)", + "category": "git", + "command": "git status", + "exit_code": 0, + "filter": "noise-strip", + "baseline_chars": 529, + "eco_chars": 331, + "baseline_tokens": 143, + "eco_tokens": 91, + "saved_tokens": 52, + "saved_pct": 36.4, + "baseline_tokens_chars4": 133, + "eco_tokens_chars4": 83 + }, + { + "label": "cat-large", + "op": "cat (900-line source file)", + "category": "listing", + "command": "cat src/tool_system/tools/bash/bash_tool.py", + "exit_code": 0, + "filter": "head-cap", + "baseline_chars": 30031, + "eco_chars": 2169, + "baseline_tokens": 6833, + "eco_tokens": 552, + "saved_tokens": 6281, + "saved_pct": 91.9, + "baseline_tokens_chars4": 7508, + "eco_tokens_chars4": 543 + }, + { + "label": "find-py", + "op": "find . -name '*.py'", + "category": "listing", + "command": "find . -type f -name '*.py' -not -path './.git/*'", + "exit_code": 0, + "filter": "head-cap", + "baseline_chars": 30031, + "eco_chars": 2321, + "baseline_tokens": 7631, + "eco_tokens": 600, + "saved_tokens": 7031, + "saved_pct": 92.1, + "baseline_tokens_chars4": 7508, + "eco_tokens_chars4": 581 + }, + { + "label": "git-diff-large", + "op": "git diff v1.0.0..v1.1.0 -- src", + "category": "listing", + "command": "git diff v1.0.0..v1.1.0 -- src", + "exit_code": 0, + "filter": "head-cap", + "baseline_chars": 30033, + "eco_chars": 2548, + "baseline_tokens": 7561, + "eco_tokens": 748, + "saved_tokens": 6813, + "saved_pct": 90.1, + "baseline_tokens_chars4": 7509, + "eco_tokens_chars4": 637 + }, + { + "label": "git-log-full", + "op": "git log -n 300 (full format)", + "category": "listing", + "command": "git log -n 300", + "exit_code": 0, + "filter": "head-cap", + "baseline_chars": 30032, + "eco_chars": 3688, + "baseline_tokens": 7714, + "eco_tokens": 946, + "saved_tokens": 6768, + "saved_pct": 87.7, + "baseline_tokens_chars4": 7508, + "eco_tokens_chars4": 922 + }, + { + "label": "git-log-oneline", + "op": "git log --oneline (full history)", + "category": "listing", + "command": "git log --oneline", + "exit_code": 0, + "filter": "head-cap", + "baseline_chars": 30031, + "eco_chars": 5298, + "baseline_tokens": 8758, + "eco_tokens": 1549, + "saved_tokens": 7209, + "saved_pct": 82.3, + "baseline_tokens_chars4": 7508, + "eco_tokens_chars4": 1325 + }, + { + "label": "grep-defs", + "op": "grep -rn 'def ' (subsystem sweep)", + "category": "listing", + "command": "grep -rn 'def ' src/tool_system/", + "exit_code": 0, + "filter": null, + "baseline_chars": 30030, + "eco_chars": 30030, + "baseline_tokens": 7697, + "eco_tokens": 7697, + "saved_tokens": 0, + "saved_pct": 0.0, + "baseline_tokens_chars4": 7508, + "eco_tokens_chars4": 7508 + }, + { + "label": "grep-defs-wide", + "op": "grep -rn 'def ' src/ (repo-wide)", + "category": "listing", + "command": "grep -rn 'def ' src/", + "exit_code": 0, + "filter": "head-cap", + "baseline_chars": 30032, + "eco_chars": 4618, + "baseline_tokens": 7582, + "eco_tokens": 1219, + "saved_tokens": 6363, + "saved_pct": 83.9, + "baseline_tokens_chars4": 7508, + "eco_tokens_chars4": 1155 + }, + { + "label": "ls-R", + "op": "ls -R src (whole tree)", + "category": "listing", + "command": "ls -R src", + "exit_code": 0, + "filter": "head-cap", + "baseline_chars": 30030, + "eco_chars": 791, + "baseline_tokens": 9088, + "eco_tokens": 225, + "saved_tokens": 8863, + "saved_pct": 97.5, + "baseline_tokens_chars4": 7508, + "eco_tokens_chars4": 198 + }, + { + "label": "log-dedup", + "op": "log show --last 90s (system log)", + "category": "logs", + "command": "log show --last 90s --style syslog", + "exit_code": 0, + "filter": "log-dedup", + "baseline_chars": 30033, + "eco_chars": 4841, + "baseline_tokens": 10512, + "eco_tokens": 1977, + "saved_tokens": 8535, + "saved_pct": 81.2, + "baseline_tokens_chars4": 7509, + "eco_tokens_chars4": 1211 + }, + { + "label": "ruff-check", + "op": "ruff check (findings)", + "category": "lint", + "command": "ruff check .", + "exit_code": 1, + "filter": null, + "baseline_chars": 683, + "eco_chars": 683, + "baseline_tokens": 192, + "eco_tokens": 192, + "saved_tokens": 0, + "saved_pct": 0.0, + "baseline_tokens_chars4": 171, + "eco_tokens_chars4": 171 + }, + { + "label": "docker-ps", + "op": "docker ps", + "category": "passthrough", + "command": "docker ps", + "exit_code": 0, + "filter": null, + "baseline_chars": 70, + "eco_chars": 70, + "baseline_tokens": 16, + "eco_tokens": 16, + "saved_tokens": 0, + "saved_pct": 0.0, + "baseline_tokens_chars4": 18, + "eco_tokens_chars4": 18 + }, + { + "label": "git-status-clean", + "op": "git status (clean tree)", + "category": "passthrough", + "command": "git status", + "exit_code": 0, + "filter": null, + "baseline_chars": 151, + "eco_chars": 151, + "baseline_tokens": 36, + "eco_tokens": 36, + "saved_tokens": 0, + "saved_pct": 0.0, + "baseline_tokens_chars4": 38, + "eco_tokens_chars4": 38 + }, + { + "label": "wc-readme", + "op": "wc -l README.md", + "category": "passthrough", + "command": "wc -l README.md", + "exit_code": 0, + "filter": null, + "baseline_chars": 18, + "eco_chars": 18, + "baseline_tokens": 5, + "eco_tokens": 5, + "saved_tokens": 0, + "saved_pct": 0.0, + "baseline_tokens_chars4": 5, + "eco_tokens_chars4": 5 + } + ], + "corpus_total": { + "baseline": 92989, + "eco": 17767 + }, + "filter_hits_total": { + "baseline": 84622, + "eco": 9400 + }, + "rtk_session_projection": { + "rows": [ + { + "op": "`ls` / `tree`", + "freq": 10, + "standard": 2000, + "rtk": 400, + "eco": 2000, + "eco_pct": 0.0 + }, + { + "op": "`cat` / read", + "freq": 20, + "standard": 40000, + "rtk": 12000, + "eco": 40000, + "eco_pct": 0.0 + }, + { + "op": "`grep` / `rg`", + "freq": 8, + "standard": 16000, + "rtk": 3200, + "eco": 16000, + "eco_pct": 0.0 + }, + { + "op": "`git status`", + "freq": 10, + "standard": 3000, + "rtk": 600, + "eco": 1908, + "eco_pct": 36.0 + }, + { + "op": "`git diff`", + "freq": 5, + "standard": 10000, + "rtk": 2500, + "eco": 10000, + "eco_pct": 0.0 + }, + { + "op": "`git log`", + "freq": 5, + "standard": 2500, + "rtk": 500, + "eco": 2500, + "eco_pct": 0.0 + }, + { + "op": "`git add/commit/push`", + "freq": 8, + "standard": 1600, + "rtk": 120, + "eco": 1007, + "eco_pct": 37.0 + }, + { + "op": "`cargo test` / `npm test`", + "freq": 5, + "standard": 25000, + "rtk": 2500, + "eco": 9850, + "eco_pct": 60.0 + }, + { + "op": "`ruff check`", + "freq": 3, + "standard": 3000, + "rtk": 600, + "eco": 3000, + "eco_pct": 0.0 + }, + { + "op": "`pytest`", + "freq": 4, + "standard": 8000, + "rtk": 800, + "eco": 2320, + "eco_pct": 71.0 + }, + { + "op": "`go test`", + "freq": 3, + "standard": 6000, + "rtk": 600, + "eco": 6000, + "eco_pct": 0.0 + }, + { + "op": "`docker ps`", + "freq": 3, + "standard": 900, + "rtk": 180, + "eco": 900, + "eco_pct": 0.0 + } + ], + "standard": 118000, + "rtk_claimed": 23900, + "eco": 95485 + } +} \ No newline at end of file diff --git a/eval/eco/results/results.md b/eval/eco/results/results.md new file mode 100644 index 000000000..04947a011 --- /dev/null +++ b/eval/eco/results/results.md @@ -0,0 +1,59 @@ +# /eco benchmark results + +Generated by `eval/eco/measure.py` over the corpus captured by +`eval/eco/capture_corpus.py`. Tokens are tiktoken `cl100k_base` counts +of the exact wire content (baseline vs eco rendering, including the +recovery-hint line and `returnCodeInterpretation` suffix). + +## Per-operation (measured on real outputs) + +| Operation | Filter | Raw tokens | /eco tokens | Saved | +|---|---|---:|---:|---:| +| go test ./... (failing run) | — (passthrough) | 322 | 322 | **0%** | +| go test -v ./... (failing run) | go-test | 527 | 227 | **-56%** | +| npx jest (failing run) | jest | 444 | 175 | **-60%** | +| npx jest (green run) | — (passthrough) | 50 | 50 | **0%** | +| pytest (failing run) | pytest | 1,347 | 390 | **-71%** | +| pytest -v (failing run) | pytest | 1,925 | 392 | **-79%** | +| pytest (repo suite slice, all pass) | pytest | 170 | 63 | **-62%** | +| pytest -v (green run) | pytest | 359 | 60 | **-83%** | +| npm install jest | noise-strip | 188 | 8 | **-95%** | +| pip install flask | noise-strip | 514 | 85 | **-83%** | +| git clone --progress | noise-strip | 6,868 | 18 | **-99%** | +| git commit | — (passthrough) | 49 | 49 | **0%** | +| git push --progress | noise-strip | 6,458 | 75 | **-98%** | +| git status (3 modified, 2 untracked) | noise-strip | 143 | 91 | **-36%** | +| cat (900-line source file) | head-cap | 6,833 | 552 | **-91%** | +| find . -name '*.py' | head-cap | 7,631 | 600 | **-92%** | +| git diff v1.0.0..v1.1.0 -- src | head-cap | 7,561 | 748 | **-90%** | +| git log -n 300 (full format) | head-cap | 7,714 | 946 | **-87%** | +| git log --oneline (full history) | head-cap | 8,758 | 1,549 | **-82%** | +| grep -rn 'def ' (subsystem sweep) | — (passthrough) | 7,697 | 7,697 | **0%** | +| grep -rn 'def ' src/ (repo-wide) | head-cap | 7,582 | 1,219 | **-83%** | +| ls -R src (whole tree) | head-cap | 9,088 | 225 | **-97%** | +| log show --last 90s (system log) | log-dedup | 10,512 | 1,977 | **-81%** | +| ruff check (findings) | — (passthrough) | 192 | 192 | **0%** | +| docker ps | — (passthrough) | 16 | 16 | **0%** | +| git status (clean tree) | — (passthrough) | 36 | 36 | **0%** | +| wc -l README.md | — (passthrough) | 5 | 5 | **0%** | +| **Corpus total** | | **92,989** | **17,767** | **-80%** | + +Filter-hit subset: 84,622 → 9,400 tokens (**-88%**) across 19 operations; 8 operations passed through unchanged (0%). + +## RTK 30-minute-session model, recomputed with measured ratios + +| Operation | Freq | Standard | rtk (claimed) | clawcodex /eco (measured) | +|---|---:|---:|---:|---:| +| `ls` / `tree` | 10x | 2,000 | 400 | 2,000 (0%) | +| `cat` / read | 20x | 40,000 | 12,000 | 40,000 (0%) | +| `grep` / `rg` | 8x | 16,000 | 3,200 | 16,000 (0%) | +| `git status` | 10x | 3,000 | 600 | 1,908 (-36%) | +| `git diff` | 5x | 10,000 | 2,500 | 10,000 (0%) | +| `git log` | 5x | 2,500 | 500 | 2,500 (0%) | +| `git add/commit/push` | 8x | 1,600 | 120 | 1,007 (-37%) | +| `cargo test` / `npm test` | 5x | 25,000 | 2,500 | 9,850 (-60%) | +| `ruff check` | 3x | 3,000 | 600 | 3,000 (0%) | +| `pytest` | 4x | 8,000 | 800 | 2,320 (-71%) | +| `go test` | 3x | 6,000 | 600 | 6,000 (0%) | +| `docker ps` | 3x | 900 | 180 | 900 (0%) | +| **Total** | | **~118,000** | **~23,900 (-80%)** | **~95,485 (-19%)** | diff --git a/src/eco/filters.py b/src/eco/filters.py index a574e7020..a06cd30d6 100644 --- a/src/eco/filters.py +++ b/src/eco/filters.py @@ -395,7 +395,9 @@ def filter_test_runner(command: str, exit_code: int, text: str) -> FilterHit | N ( re.compile(r"(^|[\s;&|(])(npm|npx|yarn|pnpm|bun)\b"), ( - re.compile(r"^npm (WARN|notice)\b"), + # npm >=9 lowercased its log prefixes ("npm warn deprecated ..."); + # error lines are deliberately NOT ceremony and must survive. + re.compile(r"^npm (WARN|warn|notice)\b"), re.compile(r"^\s*reify:"), re.compile(r"^Progress: resolved \d+"), *_PROGRESS_BAR_PATTERNS, diff --git a/tests/test_eco.py b/tests/test_eco.py index 745e6e921..fe4c1e614 100644 --- a/tests/test_eco.py +++ b/tests/test_eco.py @@ -414,6 +414,23 @@ def test_noise_strip_preserves_errors(): assert "ERROR: No matching distribution found" in hit.body +def test_noise_strip_npm_lowercase_warn(): + """npm >=9 lowercased its log prefixes ("npm warn deprecated ..."); + deprecation spam is ceremony, but `npm error` lines must survive.""" + text = ( + "added 294 packages in 6s\n" + "npm warn deprecated inflight@1.0.6: This module is not supported\n" + "npm WARN deprecated abab@2.0.6: Use your platform's native atob()\n" + "npm warn deprecated glob@7.2.3: Old versions of glob\n" + "npm error code ELIFECYCLE\n" + ) + hit = filter_noise_strip("npm install --no-fund jest", 0, text) + assert hit is not None and hit.safe_loss + assert "added 294 packages" in hit.body + assert "deprecated" not in hit.body # both npm<9 WARN and npm>=9 warn drop + assert "npm error code ELIFECYCLE" in hit.body + + # ── filters: log_dedup ───────────────────────────────────────────────────────