Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions docs/reference/trajectory-assessment.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,13 @@ the frontend does not present an unspecified or other-language result as a new
analysis in the selected language. Output-language metadata records the request,
not independent verification of a model's translation or judgments.

The Codex observer reuses native Chat's shared authenticated HTTP transport.
Credentials and refresh remain with Codex's existing signed-in account; the
observer neither copies tokens nor imports user tool/rule configuration. Its
explicit read-only sandbox, disabled shell/exec/web tools and fixed supplied
evidence remain in force. A failed provider attempt stays in job history and
requires an explicit retry after the host problem is repaired.

## What the measurements should answer

Use one **research episode** as the process unit: a bounded question or competing
Expand Down
10 changes: 10 additions & 0 deletions loopx/capabilities/benchmark_toolkit/trajectory_assessment.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from typing import Any

from ...control_plane.effect_runtime import effect_runtime_result
from ..native_chat.codex_context import shared_chatgpt_transport


def _digest(value: Any) -> str:
Expand Down Expand Up @@ -144,12 +145,21 @@ def execute(
_write(directory / "output-schema.json", prepared["output_schema"])
(directory / "prompt.txt").write_text(prepared["prompt"], encoding="utf-8")
config = prepared["config"]
# Keep native account routing while ignoring unrelated user tools and
# rules. The shared host transport delegates credentials/refresh to
# Codex; it supplies no endpoint, bearer token or account configuration.
transport = shared_chatgpt_transport({}, {})
provider = transport["model_provider"]
transport_args = ["-c", f"model_provider={json.dumps(provider)}"]
for key, value in transport["model_providers"][provider].items():
transport_args.extend(["-c", f"model_providers.{provider}.{key}={json.dumps(value)}"])
command = [
codex,
"exec",
"--ephemeral",
"--ignore-user-config",
"--ignore-rules",
*transport_args,
"--skip-git-repo-check",
"--sandbox",
"read-only",
Expand Down
9 changes: 9 additions & 0 deletions tests/capabilities/test_trajectory_assessment.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,15 @@ def provider(
import json,sys,time
from pathlib import Path
assert "\\u539f\\u59cb\\u76ee\\u6807" in sys.stdin.buffer.read().decode("utf-8")
options = dict(arg.split('=', 1) for index, arg in enumerate(sys.argv) if index > 0 and sys.argv[index-1] == '-c')
selected = json.loads(options['model_provider'])
assert json.loads(options['model_providers.' + selected + '.requires_openai_auth']) is True
assert json.loads(options['model_providers.' + selected + '.supports_websockets']) is False
assert not any(key.endswith(('.base_url', '.env_key', '.experimental_bearer_token')) for key in options)
assert '--ignore-user-config' in sys.argv and '--ignore-rules' in sys.argv
assert sys.argv[sys.argv.index('--sandbox')+1] == 'read-only'
assert json.loads(options['web_search']) == 'disabled'
assert 'shell_tool' in sys.argv and 'unified_exec' in sys.argv
time.sleep({delay})
if {code}: sys.exit({code})
Path(sys.argv[sys.argv.index('-o')+1]).write_bytes({payload!r})
Expand Down
Loading