From 2f14366a06e299e601c984de7e3e659a8bc76ba6 Mon Sep 17 00:00:00 2001 From: loegaire Date: Wed, 16 Sep 2026 08:42:35 -0400 Subject: [PATCH] feat(sleep): harvest Codex CLI rollout sessions (sessions/YYYY/MM/DD) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit harvest_codex() only walked the flat Codex Desktop archive (~/.codex/archived_sessions), so machines running the Codex CLI — which writes nested ~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl files — harvested zero sessions, and the 'auto' source silently fell back to Claude transcripts. - walk both layouts: the Desktop archive (flat) and the sibling sessions/ tree (recursive) - deduplicate adjacent (role, text) pairs when digesting: CLI rollouts record every turn twice (response_item/message + event_msg/*_message, adjacent in either order), which double-counted turns - add tests/test_harvest_codex_cli.py covering nested-layout discovery, desktop+CLI combination, and the dedup/environment_context cases --- skillopt_sleep/harvest_codex.py | 56 +++++++++++++---- tests/test_harvest_codex_cli.py | 107 ++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 12 deletions(-) create mode 100644 tests/test_harvest_codex_cli.py diff --git a/skillopt_sleep/harvest_codex.py b/skillopt_sleep/harvest_codex.py index c50a237cb..defa9a146 100644 --- a/skillopt_sleep/harvest_codex.py +++ b/skillopt_sleep/harvest_codex.py @@ -1,8 +1,13 @@ -"""SkillOpt-Sleep Codex Desktop session harvesting. +"""SkillOpt-Sleep Codex session harvesting. -Reads Codex Desktop archived session JSONL files and normalizes them into -``SessionDigest`` records without copying developer/system instructions, tool -arguments, or raw tool outputs. +Reads Codex session JSONL files and normalizes them into ``SessionDigest`` +records without copying developer/system instructions, tool arguments, or raw +tool outputs. Two layouts are supported and harvested together: + +- Codex Desktop: flat ``~/.codex/archived_sessions/*.jsonl`` files. +- Codex CLI: ``~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl`` rollout files + (each turn appears twice, as a ``response_item/message`` and an + ``event_msg/*_message`` record; the digest deduplicates adjacent pairs). """ from __future__ import annotations @@ -132,6 +137,9 @@ def digest_codex_archived_session(path: str, project: str = "") -> Optional[Sess feedback: List[str] = [] n_user = 0 n_asst = 0 + # Codex CLI rollout files record every turn twice (response_item/message + # and event_msg/*_message, adjacent in either order); skip the repeat. + last_recorded = ("", "") for rec in _iter_jsonl(path): payload = _payload(rec) @@ -169,6 +177,9 @@ def digest_codex_archived_session(path: str, project: str = "") -> Optional[Sess sanitized = _sanitize_text(text) if not sanitized: continue + if (output_role, sanitized) == last_recorded: + continue + last_recorded = (output_role, sanitized) if output_role == "user": n_user += 1 user_prompts.append(sanitized) @@ -198,6 +209,31 @@ def digest_codex_archived_session(path: str, project: str = "") -> Optional[Sess ) +def _candidate_paths(archived_sessions_dir: str) -> List[str]: + """List Codex session JSONL files across both on-disk layouts. + + ``archived_sessions_dir`` is the flat Codex Desktop store; the sibling + ``sessions/`` tree (``/sessions/YYYY/MM/DD/rollout-*.jsonl``) + is what the Codex CLI writes and is walked recursively. + """ + paths: List[str] = [] + if os.path.isdir(archived_sessions_dir): + paths.extend( + os.path.join(archived_sessions_dir, fn) + for fn in os.listdir(archived_sessions_dir) + if fn.endswith(".jsonl") + ) + sessions_dir = os.path.join( + os.path.dirname(os.path.abspath(archived_sessions_dir)), "sessions" + ) + if os.path.isdir(sessions_dir): + for root, _dirs, files in os.walk(sessions_dir): + paths.extend( + os.path.join(root, fn) for fn in files if fn.endswith(".jsonl") + ) + return paths + + def harvest_codex( archived_sessions_dir: str, *, @@ -206,16 +242,12 @@ def harvest_codex( since_iso: Optional[str] = None, limit: int = 0, ) -> List[SessionDigest]: - """Walk ``~/.codex/archived_sessions`` and return matching digests.""" + """Walk ``~/.codex/archived_sessions`` + ``~/.codex/sessions`` and return + matching digests.""" digests: List[SessionDigest] = [] - if not os.path.isdir(archived_sessions_dir): + paths = _candidate_paths(archived_sessions_dir) + if not paths: return digests - - paths = [ - os.path.join(archived_sessions_dir, fn) - for fn in os.listdir(archived_sessions_dir) - if fn.endswith(".jsonl") - ] paths.sort(key=lambda p: os.path.getmtime(p), reverse=True) project_hint = invoked_project if scope == "invoked" else "" diff --git a/tests/test_harvest_codex_cli.py b/tests/test_harvest_codex_cli.py new file mode 100644 index 000000000..9fbe7e4fa --- /dev/null +++ b/tests/test_harvest_codex_cli.py @@ -0,0 +1,107 @@ +"""Tests for harvesting Codex CLI rollout sessions (sessions/YYYY/MM/DD).""" +from __future__ import annotations + +import json + +from skillopt_sleep.harvest_codex import digest_codex_archived_session, harvest_codex + + +def _rec(ts, rtype, payload): + return {"timestamp": ts, "type": rtype, "payload": payload} + + +def _write_rollout(path, records): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + for rec in records: + f.write(json.dumps(rec) + "\n") + return str(path) + + +def _cli_rollout_records(): + return [ + _rec("2026-07-18T10-11-28.000Z", "session_meta", { + "session_id": "abc", "cwd": "/home/u/proj", + }), + _rec("2026-07-18T10-11-29.000Z", "response_item", { + "type": "message", "role": "developer", + "content": [{"type": "input_text", "text": "developer instructions"}], + }), + _rec("2026-07-18T10-11-30.000Z", "response_item", { + "type": "message", "role": "user", + "content": [{"type": "input_text", "text": + "\n /home/u/proj\n"}], + }), + _rec("2026-07-18T10-11-31.000Z", "response_item", { + "type": "message", "role": "user", + "content": [{"type": "input_text", "text": "fix the failing tests"}], + }), + # CLI rollouts repeat each user turn as event_msg/user_message. + _rec("2026-07-18T10-11-32.000Z", "event_msg", { + "type": "user_message", "message": "fix the failing tests", + }), + # Assistant turn: event_msg first, response_item second. + _rec("2026-07-18T10-11-33.000Z", "event_msg", { + "type": "agent_message", "message": "Running the suite now.", + }), + _rec("2026-07-18T10-11-34.000Z", "response_item", { + "type": "message", "role": "assistant", + "content": [{"type": "output_text", "text": "Running the suite now."}], + }), + _rec("2026-07-18T10-12-00.000Z", "response_item", { + "type": "exec_command_end", + }), + _rec("2026-07-18T10-12-30.000Z", "response_item", { + "type": "message", "role": "user", + "content": [{"type": "input_text", "text": "wrong flag, try again"}], + }), + _rec("2026-07-18T10-12-31.000Z", "event_msg", { + "type": "user_message", "message": "wrong flag, try again", + }), + ] + + +def test_harvest_finds_cli_rollout_sessions(tmp_path): + codex_home = tmp_path / "codex" + rollout = codex_home / "sessions" / "2026" / "07" / "18" / ( + "rollout-2026-07-18T10-11-28-abc.jsonl" + ) + _write_rollout(rollout, _cli_rollout_records()) + + digests = harvest_codex(str(codex_home / "archived_sessions"), scope="all") + assert len(digests) == 1 + d = digests[0] + assert d.project == "/home/u/proj" + # duplicated turns counted once; environment_context dropped + assert d.n_user_turns == 2 + assert d.user_prompts == ["fix the failing tests", "wrong flag, try again"] + assert d.tools_used == ["exec_command"] + assert d.started_at.startswith("2026-07-18T10-11-28") + assert d.ended_at.startswith("2026-07-18T10-12-31") + + +def test_harvest_combines_desktop_and_cli_layouts(tmp_path): + codex_home = tmp_path / "codex" + _write_rollout( + codex_home / "sessions" / "2026" / "07" / "18" / "rollout-cli.jsonl", + _cli_rollout_records(), + ) + archived = codex_home / "archived_sessions" + _write_rollout(archived / "desktop-session.jsonl", [ + _rec("2026-06-01T09:00:00Z", "session", {"cwd": "/home/u/other"}), + _rec("2026-06-01T09:00:01Z", "payload", { + "type": "user_message", "message": "desktop prompt", + }), + ]) + + digests = harvest_codex(str(archived), scope="all") + assert {d.project for d in digests} == {"/home/u/proj", "/home/u/other"} + + +def test_digest_returns_none_without_messages(tmp_path): + path = _write_rollout( + tmp_path / "rollout-empty.jsonl", + [_rec("2026-07-18T10-11-28.000Z", "session_meta", + {"session_id": "abc", "cwd": "/home/u/proj"})], + ) + assert digest_codex_archived_session(path) is None