From 76dcc78417874ea27db31682ac8f5a6f5d544d31 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sat, 5 Sep 2026 17:25:53 -0400 Subject: [PATCH 01/35] Implemented: enable_bottom_toolbar=True now keeps the toolbar visible and refreshing during command execution. Prompts, pagers, and shell commands temporarily suspend it while using the terminal. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Try the work command examples/getting_started.py for a demonstration. The feature uses two prompt-toolkit displays at different times: the existing PromptSession while waiting for input, and a dedicated CommandToolbar while commands execute. - Command execution stays on the main thread. The command loop starts the toolbar’s UI in a background thread, using the same content callback, styling, and refresh interval. - Output appears above the toolbar. Stable stream wrappers route Python and piped subprocess output through prompt-toolkit’s output proxy. Their identities remain consistent across cmd2 redirection and toolbar suspension. - Input remains coordinated. The toolbar handles terminal position reports, saves typed-ahead keys for the next prompt, and forwards Ctrl-C to the main thread. - Other terminal interfaces get exclusive access. Nested prompts, pagers, and shell commands temporarily suspend the toolbar. Custom commands can do the same with suspend_bottom_toolbar(). - Cleanup restores normal terminal operation. When execution ends, buffered output is flushed, workers are stopped, and the original streams are restored. Most supporting code lives in cmd2/command_toolbar.py, with lifecycle integration in cmd2/cmd2.py. Because get_bottom_toolbar() runs in the UI thread during commands, shared mutable state should be protected with a lock. --- CHANGELOG.md | 6 + cmd2/cmd2.py | 74 +++++++- cmd2/command_toolbar.py | 271 +++++++++++++++++++++++++++++ docs/features/prompt.md | 32 +++- examples/getting_started.py | 8 + mkdocs.yml | 1 + tests/test_command_toolbar.py | 319 ++++++++++++++++++++++++++++++++++ 7 files changed, 698 insertions(+), 13 deletions(-) create mode 100644 cmd2/command_toolbar.py create mode 100644 tests/test_command_toolbar.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 55965220c..870acc218 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 4.3.0 (TBD) + +- Enhancements + - `enable_bottom_toolbar=True` now keeps the toolbar visible and refreshing during command + execution + ## 4.2.3 (September 2, 2026) - Bug Fixes diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index 5ceffb085..432bc9f22 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -48,6 +48,7 @@ from collections.abc import ( Callable, Iterable, + Iterator, Mapping, Sequence, ) @@ -106,6 +107,7 @@ from . import ( argparse_completer, argparse_utils, + command_toolbar, constants, plugin, utils, @@ -416,7 +418,7 @@ def __init__( This allows CommandSets with custom constructor parameters to be loaded. This also allows the a set of CommandSets to be provided when `auto_load_commands` is set to False - :param enable_bottom_toolbar: if ``True``, enables a bottom toolbar while at the main prompt. + :param enable_bottom_toolbar: if ``True``, enables a bottom toolbar at the main prompt and during commands. Override ``get_bottom_toolbar()`` to define its content. :param enable_rprompt: if ``True``, enables a right prompt while at the main prompt. Override ``get_rprompt()`` to define its content. @@ -557,6 +559,7 @@ def __init__( # custom prompt). Completion and UI logic should reference this variable # to ensure they modify the correct session state. self.active_session = self.main_session + self._command_toolbar: command_toolbar.CommandToolbar | None = None # Commands to exclude from the history command self.exclude_from_history = ["_eof", "history"] @@ -1868,6 +1871,7 @@ def pfeedback( rich_print_kwargs=rich_print_kwargs, ) + @command_toolbar.suspend_toolbar def ppaged( self, *objects: Any, @@ -2042,7 +2046,7 @@ def ppretty( def get_bottom_toolbar(self) -> AnyFormattedText: """Get the bottom toolbar content. - This method is called by prompt-toolkit while at the main prompt if ``enable_bottom_toolbar`` + This method is called by prompt-toolkit at the main prompt and during commands if ``enable_bottom_toolbar`` was set to ``True`` during initialization. Because prompt-toolkit executes this callback on every UI refresh (such as on every keypress or at scheduled refresh intervals), keeping this function highly optimized is critical to ensuring the CLI remains responsive. @@ -2051,10 +2055,50 @@ def get_bottom_toolbar(self) -> AnyFormattedText: your application. This could be information like the application name, current state, or even a real-time clock. + During command execution this callback runs in a background UI thread. Protect shared + state with a lock when necessary. The toolbar is suspended while another prompt, pager, + or interactive shell owns the terminal. + :return: Content to populate the bottom toolbar. """ return None + @contextlib.contextmanager + def suspend_bottom_toolbar(self) -> Iterator[None]: + """Temporarily hide the command toolbar and give exclusive access to the terminal. + + Use this context manager around application-specific calls to ``input()``, other + terminal UIs, or subprocesses that inherit the terminal. cmd2 automatically suspends + its toolbar for its own input prompts, pagers, and shell commands. + """ + if self._command_toolbar is None: + yield + else: + with self._command_toolbar.suspend(): + yield + + @contextlib.contextmanager + def _command_toolbar_context(self) -> Iterator[None]: + """Display the toolbar around commands launched by the interactive command loop.""" + if ( + self._command_toolbar is not None + or self.main_session.bottom_toolbar is None + or not self._is_tty_session(self.main_session) + ): + yield + return + + toolbar = command_toolbar.CommandToolbar(self) + try: + with self.sigint_protection: + toolbar.start() + self._command_toolbar = toolbar + yield + finally: + with self.sigint_protection: + toolbar.stop() + self._command_toolbar = None + def get_rprompt(self) -> AnyFormattedText: """Provide text to populate the prompt-toolkit right prompt. @@ -3079,6 +3123,7 @@ def onecmd_plus_hooks( return stop + @command_toolbar.suspend_toolbar def _run_cmdfinalization_hooks(self, stop: bool, statement: Statement | None) -> bool: """Run the command finalization hooks.""" if self._initial_termios_settings is not None and self.stdin.isatty(): # type: ignore[unreachable] @@ -3314,12 +3359,17 @@ def _redirect_output(self, statement: Statement) -> utils.RedirectionSavedState: if shell: kwargs["executable"] = shell - # For any stream that is a StdSim, we will use a pipe so we can capture its output + # Capture subprocess output when it must pass through a Python stream, + # including the toolbar proxy which prints above the running display. proc = subprocess.Popen( # noqa: S602 statement.redirect_to, stdin=subproc_stdin, - stdout=subprocess.PIPE if isinstance(self.stdout, utils.StdSim) else self.stdout, # type: ignore[unreachable] - stderr=subprocess.PIPE if isinstance(sys.stderr, utils.StdSim) else sys.stderr, + stdout=subprocess.PIPE + if isinstance(self.stdout, (utils.StdSim, command_toolbar.ToolbarStream)) # type: ignore[unreachable] + else self.stdout, + stderr=subprocess.PIPE + if isinstance(sys.stderr, (utils.StdSim, command_toolbar.ToolbarStream)) + else sys.stderr, shell=True, **kwargs, ) @@ -3515,6 +3565,7 @@ def _is_tty_session(session: PromptSession[str]) -> bool: # a DummyOutput. return not isinstance(session.input, DummyInput) + @command_toolbar.suspend_toolbar def _read_raw_input( self, prompt: Callable[[], ANSI | str] | ANSI | str, @@ -3796,7 +3847,11 @@ def _cmdloop(self) -> None: """ try: # Run startup commands - stop = self.runcmds_plus_hooks(self._startup_commands) + if self._startup_commands: + with self._command_toolbar_context(): + stop = self.runcmds_plus_hooks(self._startup_commands) + else: + stop = False self._startup_commands.clear() while not stop: @@ -3810,7 +3865,8 @@ def _cmdloop(self) -> None: line = "_eof" # Run the command along with all associated pre and post hooks - stop = self.onecmd_plus_hooks(line) + with self._command_toolbar_context(): + stop = self.onecmd_plus_hooks(line) finally: with self.sigint_protection: # Shut down the alert thread. @@ -4640,6 +4696,7 @@ def do_quit(self, _: argparse.Namespace) -> bool | None: self.last_result = True return True + @command_toolbar.suspend_toolbar def select(self, opts: str | Iterable[str] | Iterable[tuple[Any, str | None]], prompt: str = "Your choice? ") -> Any: """Present a menu to the user. @@ -4848,6 +4905,7 @@ def _build_shell_parser(cls) -> Cmd2ArgumentParser: # Preserve quotes since we are passing these strings to the shell @with_argparser(_build_shell_parser, preserve_quotes=True) + @command_toolbar.suspend_toolbar def do_shell(self, args: argparse.Namespace) -> None: """Execute a command as if at the OS prompt.""" import signal @@ -4966,6 +5024,7 @@ def _restore_cmd2_env(self, cmd2_env: _SavedCmd2Env) -> None: readline.set_completer(cmd2_env.completer) + @command_toolbar.suspend_toolbar def _run_python(self, *, pyscript: str | None = None) -> bool | None: """Run an interactive Python shell or execute a pyscript file. @@ -5177,6 +5236,7 @@ def _build_ipython_parser() -> Cmd2ArgumentParser: return argparse_utils.DEFAULT_ARGUMENT_PARSER(description="Run an interactive IPython shell.") @with_argparser(_build_ipython_parser) + @command_toolbar.suspend_toolbar def do_ipy(self, _: argparse.Namespace) -> bool | None: # pragma: no cover """Run an interactive IPython shell. diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py new file mode 100644 index 000000000..90ff014f1 --- /dev/null +++ b/cmd2/command_toolbar.py @@ -0,0 +1,271 @@ +"""Internal support for displaying a toolbar during synchronous commands.""" + +import codecs +import contextlib +import contextvars +import functools +import os +import signal +import sys +import threading +from collections.abc import Callable, Iterator +from typing import TYPE_CHECKING, Any, TextIO, TypeVar, cast + +from prompt_toolkit.application import Application, create_app_session +from prompt_toolkit.filters import Condition, is_done, renderer_height_is_known, to_filter +from prompt_toolkit.input.typeahead import get_typeahead, store_typeahead +from prompt_toolkit.key_binding import KeyBindings +from prompt_toolkit.key_binding.key_processor import KeyPress, KeyPressEvent +from prompt_toolkit.layout import HSplit, Layout, Window +from prompt_toolkit.layout.containers import ConditionalContainer +from prompt_toolkit.layout.controls import FormattedTextControl +from prompt_toolkit.patch_stdout import StdoutProxy +from prompt_toolkit.utils import suspend_to_background_supported + +if TYPE_CHECKING: + from .cmd2 import Cmd + +_F = TypeVar("_F", bound=Callable[..., Any]) + + +def suspend_toolbar(func: _F) -> _F: + """Give a method exclusive access to the terminal.""" + + @functools.wraps(func) + def wrapped(self: "Cmd", *args: Any, **kwargs: Any) -> Any: + with self.suspend_bottom_toolbar(): + return func(self, *args, **kwargs) + + return cast(_F, wrapped) + + +class _ContextStdoutProxy(StdoutProxy): + """Keep stdout's flush worker in the toolbar's isolated application session.""" + + def _start_write_thread(self) -> threading.Thread: + context = contextvars.copy_context() + thread = threading.Thread(target=context.run, args=(self._write_thread,), daemon=True) + thread.start() + return thread + + +class ToolbarStream: + """Keep a stable stream identity across suspensions and cmd2 redirections.""" + + def __init__(self, original: TextIO) -> None: + """Wrap a terminal stream while preserving its ordinary file attributes.""" + self.original = original + self.proxy: StdoutProxy | None = None + self.buffer = _ToolbarBuffer(self) + + def write(self, data: str) -> int: + """Write above the toolbar, or directly while the toolbar is suspended.""" + return (self.proxy or self.original).write(data) + + def flush(self) -> None: + """Flush the currently active output stream.""" + (self.proxy or self.original).flush() + + def __getattr__(self, name: str) -> Any: + """Delegate file attributes to the original terminal stream.""" + return getattr(self.original, name) + + +class _ToolbarBuffer: + """Decode subprocess output incrementally, including split Unicode characters.""" + + def __init__(self, stream: ToolbarStream) -> None: + self.stream = stream + self._decoder = codecs.getincrementaldecoder(stream.original.encoding or "utf-8")(errors="replace") + self._lock = threading.Lock() + + def write(self, data: bytes) -> int: + with self._lock: + self.stream.write(self._decoder.decode(data)) + return len(data) + + def flush(self) -> None: + self.stream.flush() + + def finish(self) -> None: + with self._lock: + self.stream.write(self._decoder.decode(b"", final=True)) + self._decoder.reset() + + +class CommandToolbar: + """Run a prompt-toolkit display in a thread while a command runs on the main thread. + + The display owns terminal input so it can receive cursor position reports. Keys + typed during execution are saved for the next prompt; Ctrl-C is sent to cmd2's + normal signal handler. Terminal output goes through prompt-toolkit's stdout + proxy so that it appears above the toolbar. + """ + + def __init__(self, cmd: "Cmd") -> None: + """Configure a command display using the main prompt's terminal and settings.""" + self.cmd = cmd + self._stack: contextlib.ExitStack | None = None + self._keys: list[KeyPress] = [] + self._error: BaseException | None = None + self._ready = threading.Event() + self._thread: threading.Thread | None = None + self._streams: list[ToolbarStream] = [] + self._proxy: StdoutProxy | None = None + + session = cmd.main_session + bindings = KeyBindings() + + @bindings.add("") + def save_key(event: KeyPressEvent) -> None: + self._keys.extend(event.key_sequence) + + @bindings.add("c-c") + def interrupt(event: KeyPressEvent) -> None: # noqa: ARG001 + # Match the terminal's normal Ctrl-C input flush: cancelled typeahead + # must not become a command when the main prompt resumes. + self._keys.clear() + if sys.platform == "win32": + # os.kill(..., SIGINT) terminates the process on Windows instead + # of dispatching Python's signal handler. + import _thread + + _thread.interrupt_main() + else: + os.kill(os.getpid(), signal.SIGINT) + + @bindings.add( + "c-z", + filter=Condition(lambda: suspend_to_background_supported() and to_filter(session.enable_suspend)()), + ) + def suspend(event: KeyPressEvent) -> None: + # This restores cooked mode before stopping the process group and + # redraws the toolbar after the process resumes. + event.app.suspend_to_background() + + self.app: Application[None] = Application( + layout=Layout( + HSplit( + [ + Window(height=0), + Window(), + ConditionalContainer( + Window( + FormattedTextControl(lambda: session.bottom_toolbar, style="class:bottom-toolbar.text"), + style="class:bottom-toolbar", + height=1, + always_hide_cursor=True, + ), + filter=~is_done & renderer_height_is_known, + ), + ] + ) + ), + input=session.input, + output=session.output, + style=session.style, + color_depth=session.color_depth, + refresh_interval=session.refresh_interval, + key_bindings=bindings, + erase_when_done=True, + after_render=lambda _: self._ready.set(), + ) + + def start(self) -> None: + """Start rendering and protect terminal output.""" + stack = contextlib.ExitStack() + self._stack = stack + self._ready.clear() + self._error = None + try: + stack.enter_context(create_app_session(input=self.app.input, output=self.app.output)) + # Only replace terminal streams. In particular, preserve redirected stderr + # and self.stdout when a nested command has redirected its output to a file. + for obj, name in ((self.cmd, "stdout"), (sys, "stdout"), (sys, "stderr")): + stream = getattr(obj, name) + if stream.isatty(): + wrapper = ToolbarStream(stream) + self._streams.append(wrapper) + setattr(obj, name, cast(TextIO, wrapper)) + stack.callback(self._restore_stream, obj, name, wrapper) + self._resume() + except BaseException: + self.stop() + raise + + @staticmethod + def _restore_stream(obj: Any, name: str, stream: ToolbarStream) -> None: + if getattr(obj, name) is stream: + setattr(obj, name, stream.original) + + def _resume(self) -> None: + self._ready.clear() + self._error = None + context = contextvars.copy_context() + + def run() -> None: + try: + self.app.run(handle_sigint=False, set_exception_handler=False) + except EOFError: + pass + except BaseException as exc: # noqa: BLE001 + # Propagate startup/render failures to the command thread. + self._error = exc + finally: + self._ready.set() + + self._thread = threading.Thread(target=context.run, args=(run,), name="cmd2-toolbar", daemon=True) + self._thread.start() + self._ready.wait() + if self._error is not None: + raise self._error + # The worker already combines queued writes. A batching sleep would also + # delay close(), which runs at each command finalization boundary. + self._proxy = _ContextStdoutProxy(raw=True, sleep_between_writes=0) + for stream in self._streams: + stream.proxy = self._proxy + + def _pause(self) -> None: + try: + if self._proxy is not None: + self._proxy.flush() + self._proxy.close() + finally: + self._proxy = None + for stream in self._streams: + stream.proxy = None + if self.app.is_running and self.app.loop is not None: + self.app.loop.call_soon_threadsafe(self.app.exit) + if self._thread is not None: + self._thread.join() + self._thread = None + # Application.run() saves its unprocessed queue before the thread + # exits. Those keys arrived after the ones handled by save_key(). + pending_keys = get_typeahead(self.app.input) + store_typeahead(self.app.input, self._keys + pending_keys) + self._keys.clear() + + def stop(self) -> None: + """Flush output, stop rendering, and restore the terminal and its streams.""" + try: + for stream in self._streams: + stream.buffer.finish() + self._pause() + finally: + if self._stack is not None: + self._stack.close() + self._stack = None + + @contextlib.contextmanager + def suspend(self) -> Iterator[None]: + """Temporarily restore ordinary terminal access, including nested suspensions.""" + if self._proxy is None: + yield + return + with self.cmd.sigint_protection: + self._pause() + try: + yield + finally: + with self.cmd.sigint_protection: + self._resume() diff --git a/docs/features/prompt.md b/docs/features/prompt.md index 0f8a09082..b1eaed2a2 100644 --- a/docs/features/prompt.md +++ b/docs/features/prompt.md @@ -61,7 +61,8 @@ example for a demonstration. ## Bottom Toolbar `cmd2` supports an optional, persistent bottom toolbar that is always visible at the bottom of the -terminal window while the application is idle and waiting for input. +terminal window while the application is waiting for input and while commands execute. Command +output appears above the toolbar. ### Enabling the Toolbar @@ -92,10 +93,10 @@ def get_bottom_toolbar(self) -> AnyFormattedText: ### Refreshing the Toolbar -Since the toolbar is rendered by `prompt-toolkit` as part of the prompt, it is naturally redrawn -whenever the prompt is refreshed. If you want the toolbar to update automatically (for example, to -display a clock), you can set `refresh_interval` in the [cmd2.Cmd.__init__][] constructor to a value -greater than 0.0. +The toolbar is rendered by `prompt-toolkit` and is naturally redrawn whenever the prompt is +refreshed. If you want the toolbar to update automatically during input and command execution (for +example, to display a clock), you can set `refresh_interval` in the [cmd2.Cmd.__init__][] +constructor to a value greater than 0.0. ```py class App(cmd2.Cmd): @@ -105,4 +106,23 @@ class App(cmd2.Cmd): See the [getting_started.py](https://github.com/python-cmd2/cmd2/blob/main/examples/getting_started.py) -example for a demonstration of this technique. +example for a demonstration of this technique. Run its `work 5` command to see the clock update +while the command prints output. + +During command execution, `get_bottom_toolbar()` runs in a background UI thread. Keep the callback +fast and use a lock when reading state that a command or another thread modifies. + +### Commands That Take Over the Terminal + +cmd2 temporarily hides the toolbar for its input prompts, pagers, Python environments, and shell +commands. It restores the toolbar when those operations finish. For custom terminal UIs, calls to +`input()`, or subprocesses that inherit the terminal, use [cmd2.Cmd.suspend_bottom_toolbar][]: + +```py +with self.suspend_bottom_toolbar(): + answer = input("Continue? ") +``` + +The command toolbar is used by the interactive command loop, including startup commands and scripts +launched from that loop. It is disabled for non-interactive input. Calls to `onecmd_plus_hooks()` +outside the command loop do not start a toolbar. diff --git a/examples/getting_started.py b/examples/getting_started.py index a1573c853..993c45d0c 100755 --- a/examples/getting_started.py +++ b/examples/getting_started.py @@ -25,6 +25,7 @@ import pathlib import sys import threading +import time from typing import Annotated from prompt_toolkit.application import get_app @@ -164,6 +165,13 @@ def get_rprompt(self) -> AnyFormattedText: text = f"cwd={current_working_directory}" return [(style, text)] + @cmd2.with_annotated + def do_work(self, seconds: int = 5) -> None: + """Simulate work while the bottom toolbar clock keeps updating.""" + for second in range(seconds): + self.poutput(f"Working: {second + 1}/{seconds}") + time.sleep(1) + @cmd2.with_annotated def do_cat( self, diff --git a/mkdocs.yml b/mkdocs.yml index d928fdcb0..f20c2aa07 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -75,6 +75,7 @@ plugins: filters: - "!^_" - "get_bottom_toolbar" + - "suspend_bottom_toolbar" merge_init_into_class: true docstring_style: sphinx docstring_section_style: spacy diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py new file mode 100644 index 000000000..a59322c31 --- /dev/null +++ b/tests/test_command_toolbar.py @@ -0,0 +1,319 @@ +"""Command toolbar lifecycle and terminal integration tests.""" + +import io +import sys +import threading +from types import SimpleNamespace +from unittest import mock + +import pytest +from prompt_toolkit.application import get_app +from prompt_toolkit.input import create_pipe_input +from prompt_toolkit.input.typeahead import get_typeahead +from prompt_toolkit.keys import Keys +from prompt_toolkit.output import DummyOutput +from prompt_toolkit.shortcuts import PromptSession + +from cmd2 import Cmd + + +class Terminal(io.StringIO): + def isatty(self) -> bool: + return True + + +class RecordingOutput(DummyOutput): + def __init__(self, stream: Terminal) -> None: + self.stdout = stream + + def write(self, data: str) -> None: + self.stdout.write(data) + + def write_raw(self, data: str) -> None: + self.stdout.write(data) + + +@pytest.fixture +def toolbar_app(): + app = Cmd(allow_cli_args=False) + output = Terminal() + app.stdout = output + with create_pipe_input() as pipe: + app.main_session = PromptSession( + input=pipe, + output=RecordingOutput(output), + bottom_toolbar="STATUS", + refresh_interval=0.01, + ) + yield app, pipe, output + + +def test_command_toolbar_refresh_and_output(toolbar_app, monkeypatch) -> None: + app, _, output = toolbar_app + refreshed = threading.Event() + state = ["BEFORE"] + threads = [] + + def toolbar(): + threads.append(threading.current_thread()) + if state[0] == "AFTER": + refreshed.set() + return state[0] + + app.main_session.bottom_toolbar = toolbar + monkeypatch.setattr(sys, "stdout", output) + original_stderr = sys.stderr + with app._command_toolbar_context(): + assert threading.current_thread() is threading.main_thread() + assert get_app() is app._command_toolbar.app + assert sys.stderr is original_stderr # Keep redirected stderr separate. + app.poutput("command output") + print("standard output", end="") # Flush unterminated output on exit. + state[0] = "AFTER" + assert refreshed.wait(2) + + assert "command output\n" in output.getvalue() + assert "standard output" in output.getvalue() + assert "AFTER" in output.getvalue() + assert all(thread is not threading.main_thread() and not thread.is_alive() for thread in threads) + assert app.stdout is output + assert sys.stdout is output + assert app._command_toolbar is None + + +def test_command_toolbar_redirected_output(toolbar_app, tmp_path) -> None: + app, _, output = toolbar_app + destination = tmp_path / "help.txt" + with app._command_toolbar_context(): + app.onecmd_plus_hooks(f'help > "{destination}"') + text = destination.read_text() + assert "Cmd2 Commands" in text + assert "STATUS" not in text + assert "Cmd2 Commands" not in output.getvalue() + + +def test_command_toolbar_redirection_survives_suspension(toolbar_app, tmp_path) -> None: + app, _, output = toolbar_app + destination = tmp_path / "output.txt" + + def command(statement, **kwargs): + app.poutput("before") + with app.suspend_bottom_toolbar(): + app.poutput("during") + app.poutput("after") + return False + + with mock.patch.object(app, "onecmd", side_effect=command), app._command_toolbar_context(): + app.onecmd_plus_hooks(f'custom > "{destination}"') + app.poutput("terminal output") + + assert destination.read_text() == "before\nduring\nafter\n" + assert "before" not in output.getvalue() + assert "during" not in output.getvalue() + assert "after" not in output.getvalue() + assert "terminal output" in output.getvalue() + assert app.stdout is output + + +def test_command_toolbar_pipe_output(toolbar_app) -> None: + app, _, output = toolbar_app + with app._command_toolbar_context(): + app.onecmd_plus_hooks(f'help | "{sys.executable}" -c "import sys; print(sys.stdin.read().upper())"') + assert "CMD2 COMMANDS" in output.getvalue() + + +def test_command_toolbar_binary_output(toolbar_app) -> None: + app, _, output = toolbar_app + data = "Unicode: πŸ˜‡\n".encode() + with app._command_toolbar_context(): + for byte in data: + app.stdout.buffer.write(bytes([byte])) + app.stdout.buffer.flush() + assert "Unicode: πŸ˜‡\n" in output.getvalue() + + +def test_command_toolbar_interrupt_uses_signal_handler(toolbar_app) -> None: + app, pipe, _ = toolbar_app + interrupted = threading.Event() + signal_target = "_thread.interrupt_main" if sys.platform == "win32" else "cmd2.command_toolbar.os.kill" + with mock.patch(signal_target, side_effect=lambda *_: interrupted.set()) as interrupt: + with app._command_toolbar_context(): + pipe.send_text("\x03") + assert interrupted.wait(2) + interrupt.assert_called_once() + + +def test_command_toolbar_interrupt_discards_cancelled_typeahead(toolbar_app) -> None: + app, pipe, _ = toolbar_app + interrupted = threading.Event() + received = threading.Event() + signal_target = "_thread.interrupt_main" if sys.platform == "win32" else "cmd2.command_toolbar.os.kill" + with mock.patch(signal_target, side_effect=lambda *_: interrupted.set()), app._command_toolbar_context(): + toolbar = app._command_toolbar + + def key_processed(_): + if "".join(key.data for key in toolbar._keys).endswith("kept\n"): + received.set() + + toolbar.app.key_processor.after_key_press += key_processed + pipe.send_text("cancelled\n\x03") + assert interrupted.wait(2) + # Input entered after the interrupt should still reach the next prompt. + pipe.send_text("kept\n") + assert received.wait(2) + + assert app._read_raw_input("Next: ", app.main_session) == "kept" + + +@pytest.mark.parametrize(("supported", "enabled"), [(True, True), (True, False), (False, True)]) +def test_command_toolbar_ctrl_z(toolbar_app, supported, enabled) -> None: + app, pipe, _ = toolbar_app + app.main_session.enable_suspend = enabled + processed = threading.Event() + with ( + mock.patch("cmd2.command_toolbar.suspend_to_background_supported", return_value=supported), + app._command_toolbar_context(), + ): + toolbar = app._command_toolbar + toolbar.app.key_processor.after_key_press += lambda _: processed.set() + with mock.patch.object(toolbar.app, "suspend_to_background") as suspend: + pipe.send_text("\x1a") + assert processed.wait(2) + if supported and enabled: + suspend.assert_called_once_with() + else: + suspend.assert_not_called() + + keys = get_typeahead(pipe) + assert [key.key for key in keys] == ([] if supported and enabled else [Keys.ControlZ]) + + +def test_command_toolbar_script_output_has_no_batching_delay(toolbar_app) -> None: + app, _, output = toolbar_app + sleep = mock.Mock() + + def command(statement, **kwargs): + app.poutput("script output") + return False + + # Observe requested sleeps instead of depending on the machine's execution speed. + with ( + mock.patch("prompt_toolkit.patch_stdout.time", SimpleNamespace(sleep=sleep)), + mock.patch.object(app, "onecmd", side_effect=command), + app._command_toolbar_context(), + ): + app.runcmds_plus_hooks(["custom"] * 10) + + assert output.getvalue().count("script output\n") == 10 + assert all(call.args[0] == 0 for call in sleep.call_args_list) + + +def test_command_toolbar_suspension_and_nested_input(toolbar_app) -> None: + app, pipe, output = toolbar_app + with app._command_toolbar_context(): + toolbar = app._command_toolbar + with app.suspend_bottom_toolbar(): + assert not toolbar.app.is_running + assert app.stdout.original is output + assert app.stdout.proxy is None + with app.suspend_bottom_toolbar(): + assert not toolbar.app.is_running + assert toolbar.app.is_running + + # Feed the nested prompt only after it has taken ownership of input. + result = app._read_raw_input("Value: ", app.main_session, pre_run=lambda: pipe.send_text("answer\n")) + assert result == "answer" + assert toolbar.app.is_running + + +def test_command_toolbar_typeahead(toolbar_app) -> None: + app, pipe, _ = toolbar_app + received = threading.Event() + with app._command_toolbar_context(): + toolbar = app._command_toolbar + + def key_processed(_): + if len(toolbar._keys) == len("next\n"): + received.set() + + toolbar.app.key_processor.after_key_press += key_processed + pipe.send_text("next\n") + assert received.wait(2) + + assert app._read_raw_input("Next: ", app.main_session) == "next" + + +def test_command_toolbar_typeahead_preserves_pending_input_order(toolbar_app) -> None: + app, pipe, _ = toolbar_app + exiting = threading.Event() + with app._command_toolbar_context(): + toolbar = app._command_toolbar + + def exit_after_first_key(_): + # Leave the remaining keys in prompt-toolkit's queue, as happens when + # input arrives just as a command finishes. + toolbar.app.exit() + exiting.set() + + toolbar.app.key_processor.after_key_press += exit_after_first_key + pipe.send_text("ab\n") + assert exiting.wait(2) + toolbar._thread.join(timeout=2) + assert not toolbar._thread.is_alive() + + assert app._read_raw_input("Next: ", app.main_session) == "ab" + + +@pytest.mark.parametrize("exception", [RuntimeError, KeyboardInterrupt, SystemExit]) +def test_command_toolbar_cleanup_on_exception(toolbar_app, exception) -> None: + app, _, output = toolbar_app + threads = [] + + def run_command(): + with app._command_toolbar_context(): + threads.append(app._command_toolbar._thread) + raise exception + + with pytest.raises(exception): + run_command() + assert len(threads) == 1 + assert not threads[0].is_alive() + assert app.stdout is output + assert app._command_toolbar is None + + +def test_command_toolbar_render_failure(toolbar_app) -> None: + app, _, output = toolbar_app + + def broken_toolbar(): + raise ValueError("broken toolbar") + + app.main_session.bottom_toolbar = broken_toolbar + with pytest.raises(ValueError, match="broken toolbar"), app._command_toolbar_context(): + pytest.fail("Command should not run after a toolbar startup failure") + assert app.stdout is output + assert app._command_toolbar is None + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_command_toolbar_headless(enabled) -> None: + app = Cmd(allow_cli_args=False, enable_bottom_toolbar=enabled) + with mock.patch("cmd2.command_toolbar.CommandToolbar") as toolbar, app._command_toolbar_context(): + toolbar.assert_not_called() + + +def test_cmdloop_runs_commands_with_toolbar(toolbar_app, monkeypatch) -> None: + app, _, _ = toolbar_app + monkeypatch.setattr(app, "_read_command_line", lambda _: "quit") + commands = [] + + def command(line, **kwargs): + assert threading.current_thread() is threading.main_thread() + assert app._command_toolbar.app.is_running + commands.append(line) + return line == "quit" + + app._startup_commands = ["startup"] + monkeypatch.setattr(app, "onecmd_plus_hooks", command) + app._cmdloop() + assert commands == ["startup", "quit"] From 1c491b1e01658c5de97d6bf76746b8180e774580 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sat, 5 Sep 2026 18:17:26 -0400 Subject: [PATCH 02/35] Various bug fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Pipes to interactive targets (cmd2.py:3366, command_toolbar.py:43) β€” the pipe process now inherits the real terminal via the new pipe_target() helper, and the toolbar suspends for the life of the pipe (held in RedirectionSavedState.toolbar_suspension, released in _restore_output). Streams with no file descriptor still fall back to PIPE. Verified under a real pty: alias | python3 -c "...isatty()" printed PIPE_ISATTY True; before the fix that command produced no output at all. 2. Ctrl-C not reaching subprocesses (command_toolbar.py:161) β€” POSIX now uses os.killpg(os.getpgrp(), SIGINT), matching what the terminal driver does. Verified under a pty with a child that reports its own SIGINT: post-fix CHILD_GOT_SIGINT appears and the child exits immediately; pre-fix it never fired and the child ran to completion. Windows keeps interrupt_main() β€” broadcasting a console control event there would hit unrelated processes on the same console. Ctrl-\ stays inert, which matches the main prompt's existing behavior; documented rather than changed. 3. Failing stop() disabling the toolbar (cmd2.py:2098) β€” _command_toolbar = None moved into its own finally. 4. Writes racing _pause() (command_toolbar.py) β€” an RLock shared by the toolbar and its ToolbarStreams serializes writes against the proxy swap. The lock is deliberately released before _thread.join(), since the toolbar thread can itself be blocked writing through a stream. 5. Post-startup failures swallowed β€” new _app_exited() detaches the streams when the display stops on its own, so output goes to the terminal instead of a dead proxy, and reports the error. Worth noting: the review's stated trigger doesn't hold β€” a get_bottom_toolbar() that raises on refresh is caught by asyncio's default handler and the thread survives. The fix targets the case where the thread genuinely dies. 6. Final flush cancelled before writing β€” does not reproduce. Application.run_async already awaits wait_for_cpr_responses() and any in-flight run_in_terminal futures before its loop closes (application.py:770-779). I built the exact scenario β€” an output with responds_to_cpr=True and a CPR future confirmed pending at pause time β€” and the trailing text arrived with and without the proposed fix. I reverted my drain implementation, which was adding a run_in_terminal round-trip and up to a second of CPR waiting at every command boundary for nothing. I kept the test as a regression guard and a two-line guard against Application.exit() raising in a loop callback if the app has already stopped. 7. Docs/example mismatch β€” do_work now uses Annotated[int, Argument(nargs="?")], so work and work 5 both work. Verified in the real app. --- cmd2/cmd2.py | 119 +++++++++++++++----------- cmd2/command_toolbar.py | 134 +++++++++++++++++++++++------ cmd2/utils.py | 3 + docs/features/prompt.md | 12 ++- examples/getting_started.py | 9 +- tests/test_command_toolbar.py | 155 +++++++++++++++++++++++++++++++++- 6 files changed, 353 insertions(+), 79 deletions(-) diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index 432bc9f22..9fa288c38 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -2096,8 +2096,12 @@ def _command_toolbar_context(self) -> Iterator[None]: yield finally: with self.sigint_protection: - toolbar.stop() - self._command_toolbar = None + try: + toolbar.stop() + finally: + # Always forget a toolbar that has been torn down. Keeping a failed + # one would disable the toolbar for the rest of the session. + self._command_toolbar = None def get_rprompt(self) -> AnyFormattedText: """Provide text to populate the prompt-toolkit right prompt. @@ -3359,37 +3363,50 @@ def _redirect_output(self, statement: Statement) -> utils.RedirectionSavedState: if shell: kwargs["executable"] = shell - # Capture subprocess output when it must pass through a Python stream, - # including the toolbar proxy which prints above the running display. - proc = subprocess.Popen( # noqa: S602 - statement.redirect_to, - stdin=subproc_stdin, - stdout=subprocess.PIPE - if isinstance(self.stdout, (utils.StdSim, command_toolbar.ToolbarStream)) # type: ignore[unreachable] - else self.stdout, - stderr=subprocess.PIPE - if isinstance(sys.stderr, (utils.StdSim, command_toolbar.ToolbarStream)) - else sys.stderr, - shell=True, - **kwargs, + # Hand the pipe process the real terminal when there is one to inherit, since it + # may be interactive. Otherwise capture its output so it can pass through a Python + # stream, including the toolbar proxy which prints above the running display. + pipe_stdout = ( + None + if isinstance(self.stdout, utils.StdSim) # type: ignore[unreachable] + else command_toolbar.pipe_target(self.stdout) ) + pipe_stderr = None if isinstance(sys.stderr, utils.StdSim) else command_toolbar.pipe_target(sys.stderr) + + with contextlib.ExitStack() as terminal_stack: + # The toolbar can neither draw nor hold the keyboard while a pipe process owns + # the terminal, so step aside until that process has finished. + if pipe_stdout is not None or pipe_stderr is not None: + terminal_stack.enter_context(self.suspend_bottom_toolbar()) + + proc = subprocess.Popen( # noqa: S602 + statement.redirect_to, + stdin=subproc_stdin, + stdout=subprocess.PIPE if pipe_stdout is None else pipe_stdout, + stderr=subprocess.PIPE if pipe_stderr is None else pipe_stderr, + shell=True, + **kwargs, + ) - # Popen was called with shell=True so the user can chain pipe commands and redirect their output - # like: !ls -l | grep user | wc -l > out.txt. But this makes it difficult to know if the pipe process - # started OK, since the shell itself always starts. Therefore, we will wait a short time and check - # if the pipe process is still running. - with contextlib.suppress(subprocess.TimeoutExpired): - proc.wait(0.2) + # Popen was called with shell=True so the user can chain pipe commands and redirect their output + # like: !ls -l | grep user | wc -l > out.txt. But this makes it difficult to know if the pipe process + # started OK, since the shell itself always starts. Therefore, we will wait a short time and check + # if the pipe process is still running. + with contextlib.suppress(subprocess.TimeoutExpired): + proc.wait(0.2) + + # Check if the pipe process already exited + if proc.returncode is not None: + subproc_stdin.close() + new_stdout.close() + raise RedirectionError(f"Pipe process exited with code {proc.returncode} before command could run") + redir_saved_state.redirecting = True + cmd_pipe_proc_reader = utils.ProcReader(proc, self.stdout, sys.stderr) - # Check if the pipe process already exited - if proc.returncode is not None: - subproc_stdin.close() - new_stdout.close() - raise RedirectionError(f"Pipe process exited with code {proc.returncode} before command could run") - redir_saved_state.redirecting = True - cmd_pipe_proc_reader = utils.ProcReader(proc, self.stdout, sys.stderr) + self.stdout = new_stdout - self.stdout = new_stdout + # Hold the suspension open until _restore_output() reaps the pipe process. + redir_saved_state.toolbar_suspension = terminal_stack.pop_all() elif statement.redirector in (constants.REDIRECTION_OVERWRITE, constants.REDIRECTION_APPEND): if statement.redirect_to: @@ -3441,29 +3458,35 @@ def _restore_output(self, statement: Statement, saved_redir_state: utils.Redirec :param statement: Statement object which contains the parsed input from the user :param saved_redir_state: contains information needed to restore state data """ - if saved_redir_state.redirecting: - # If we redirected output to the clipboard - if ( - statement.redirector in (constants.REDIRECTION_OVERWRITE, constants.REDIRECTION_APPEND) - and not statement.redirect_to - ): - self.stdout.seek(0) - write_to_paste_buffer(self.stdout.read()) + # The toolbar gets the terminal back once the pipe process is done with it. + with contextlib.ExitStack() as terminal_stack: + if saved_redir_state.toolbar_suspension is not None: + terminal_stack.callback(saved_redir_state.toolbar_suspension.close) + saved_redir_state.toolbar_suspension = None + + if saved_redir_state.redirecting: + # If we redirected output to the clipboard + if ( + statement.redirector in (constants.REDIRECTION_OVERWRITE, constants.REDIRECTION_APPEND) + and not statement.redirect_to + ): + self.stdout.seek(0) + write_to_paste_buffer(self.stdout.read()) - with contextlib.suppress(BrokenPipeError): - # Close the file or pipe that stdout was redirected to - self.stdout.close() + with contextlib.suppress(BrokenPipeError): + # Close the file or pipe that stdout was redirected to + self.stdout.close() - # Restore self.stdout - self.stdout = cast(TextIO, saved_redir_state.saved_self_stdout) + # Restore self.stdout + self.stdout = cast(TextIO, saved_redir_state.saved_self_stdout) - # Check if we need to wait for the process being piped to - if self._cur_pipe_proc_reader is not None: - self._cur_pipe_proc_reader.wait() + # Check if we need to wait for the process being piped to + if self._cur_pipe_proc_reader is not None: + self._cur_pipe_proc_reader.wait() - # These are restored regardless of whether the command redirected - self._cur_pipe_proc_reader = saved_redir_state.saved_pipe_proc_reader - self._redirecting = saved_redir_state.saved_redirecting + # These are restored regardless of whether the command redirected + self._cur_pipe_proc_reader = saved_redir_state.saved_pipe_proc_reader + self._redirecting = saved_redir_state.saved_redirecting def get_command_func(self, command: str) -> BoundCommandFunc[...] | None: """Get the bound command function for a command. diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index 90ff014f1..4db580f53 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -39,6 +39,23 @@ def wrapped(self: "Cmd", *args: Any, **kwargs: Any) -> Any: return cast(_F, wrapped) +def pipe_target(stream: Any) -> Any: + """Return the stream a pipe process can inherit, or ``None`` if its output must be captured. + + A pipe process may be interactive, such as ``less`` or ``fzf``, so it needs the real + terminal rather than a stream this process reads on its behalf. Look through a + :class:`ToolbarStream` wrapper, but only hand back a stream owning a file descriptor. + """ + if isinstance(stream, ToolbarStream): + stream = stream.original + try: + stream.fileno() + except (AttributeError, OSError): + # io.UnsupportedOperation, raised by streams like io.StringIO, subclasses OSError. + return None + return stream + + class _ContextStdoutProxy(StdoutProxy): """Keep stdout's flush worker in the toolbar's isolated application session.""" @@ -52,19 +69,24 @@ def _start_write_thread(self) -> threading.Thread: class ToolbarStream: """Keep a stable stream identity across suspensions and cmd2 redirections.""" - def __init__(self, original: TextIO) -> None: + def __init__(self, original: TextIO, lock: "threading.RLock") -> None: """Wrap a terminal stream while preserving its ordinary file attributes.""" self.original = original self.proxy: StdoutProxy | None = None + # Shared with the toolbar so a write from another thread cannot land on a proxy + # that is being closed. Such a write is accepted by the dead proxy and discarded. + self._lock = lock self.buffer = _ToolbarBuffer(self) def write(self, data: str) -> int: """Write above the toolbar, or directly while the toolbar is suspended.""" - return (self.proxy or self.original).write(data) + with self._lock: + return (self.proxy or self.original).write(data) def flush(self) -> None: """Flush the currently active output stream.""" - (self.proxy or self.original).flush() + with self._lock: + (self.proxy or self.original).flush() def __getattr__(self, name: str) -> Any: """Delegate file attributes to the original terminal stream.""" @@ -112,6 +134,8 @@ def __init__(self, cmd: "Cmd") -> None: self._thread: threading.Thread | None = None self._streams: list[ToolbarStream] = [] self._proxy: StdoutProxy | None = None + self._lock = threading.RLock() + self._pausing = False session = cmd.main_session bindings = KeyBindings() @@ -127,12 +151,19 @@ def interrupt(event: KeyPressEvent) -> None: # noqa: ARG001 self._keys.clear() if sys.platform == "win32": # os.kill(..., SIGINT) terminates the process on Windows instead - # of dispatching Python's signal handler. + # of dispatching Python's signal handler. This reaches only this + # process, so a console subprocess started by a command keeps + # running until it is waited on. import _thread _thread.interrupt_main() else: - os.kill(os.getpid(), signal.SIGINT) + # Raw mode clears ISIG, so no signal is generated for us. Signal the + # foreground process group the way the terminal driver would, since a + # command may be waiting on a subprocess that shares this group. Pipe + # processes are excluded because cmd2 starts them in their own session + # and forwards to them from sigint_handler(). + os.killpg(os.getpgrp(), signal.SIGINT) @bindings.add( "c-z", @@ -184,7 +215,7 @@ def start(self) -> None: for obj, name in ((self.cmd, "stdout"), (sys, "stdout"), (sys, "stderr")): stream = getattr(obj, name) if stream.isatty(): - wrapper = ToolbarStream(stream) + wrapper = ToolbarStream(stream, self._lock) self._streams.append(wrapper) setattr(obj, name, cast(TextIO, wrapper)) stack.callback(self._restore_stream, obj, name, wrapper) @@ -213,6 +244,7 @@ def run() -> None: self._error = exc finally: self._ready.set() + self._app_exited() self._thread = threading.Thread(target=context.run, args=(run,), name="cmd2-toolbar", daemon=True) self._thread.start() @@ -221,29 +253,81 @@ def run() -> None: raise self._error # The worker already combines queued writes. A batching sleep would also # delay close(), which runs at each command finalization boundary. - self._proxy = _ContextStdoutProxy(raw=True, sleep_between_writes=0) - for stream in self._streams: - stream.proxy = self._proxy + proxy = _ContextStdoutProxy(raw=True, sleep_between_writes=0) + with self._lock: + self._proxy = proxy + for stream in self._streams: + stream.proxy = proxy + + def _app_exited(self) -> None: + """Give the terminal back to the streams when the display stops on its own. + + ``_ready`` is set as soon as the first frame renders, so a failure after that is + never seen by the command thread waiting in ``_resume()``. The display is gone at + that point and its stdout proxy can no longer reach the terminal, so anything + written through it would be discarded without a trace. + """ + if self._pausing: + # A deliberate pause restores the streams itself, in the right order. + return + + with self._lock: + # Leave self._proxy set so that the next _pause() still drains and closes + # it. With the display gone, its worker writes to the terminal directly. + started = self._proxy is not None + for stream in self._streams: + stream.proxy = None + + # A proxy exists only once _resume() has handed startup failures to the command + # thread, so reporting here does not duplicate the exception it raises. + if started and self._error is not None: + self.cmd.perror(f"Bottom toolbar stopped after an error: {self._error!r}") + + def _exit(self) -> None: + """Stop the display unless it has already stopped on its own. + + Application.exit() raises once the result is set, and this runs later than the + check that scheduled it. Any exception here would reach the loop's default + handler, which prints a traceback over the terminal. + + Output queued before this does not need draining: Application.run_async() waits + for cursor position reports and for run_in_terminal() calls still in flight + before its loop closes. + """ + if self.app.is_running: + self.app.exit() def _pause(self) -> None: + self._pausing = True try: - if self._proxy is not None: - self._proxy.flush() - self._proxy.close() + try: + # Hold off other threads while the proxy drains so their output is never + # handed to a proxy whose worker has already stopped. Writes that arrive + # after this go straight to the terminal, still in order. + with self._lock: + try: + if self._proxy is not None: + self._proxy.flush() + self._proxy.close() + finally: + self._proxy = None + for stream in self._streams: + stream.proxy = None + finally: + # The lock is released before joining, since the toolbar thread may be + # blocked writing through a stream that is waiting on it. + if self.app.is_running and self.app.loop is not None: + self.app.loop.call_soon_threadsafe(self._exit) + if self._thread is not None: + self._thread.join() + self._thread = None + # Application.run() saves its unprocessed queue before the thread + # exits. Those keys arrived after the ones handled by save_key(). + pending_keys = get_typeahead(self.app.input) + store_typeahead(self.app.input, self._keys + pending_keys) + self._keys.clear() finally: - self._proxy = None - for stream in self._streams: - stream.proxy = None - if self.app.is_running and self.app.loop is not None: - self.app.loop.call_soon_threadsafe(self.app.exit) - if self._thread is not None: - self._thread.join() - self._thread = None - # Application.run() saves its unprocessed queue before the thread - # exits. Those keys arrived after the ones handled by save_key(). - pending_keys = get_typeahead(self.app.input) - store_typeahead(self.app.input, self._keys + pending_keys) - self._keys.clear() + self._pausing = False def stop(self) -> None: """Flush output, stop rendering, and restore the terminal and its streams.""" diff --git a/cmd2/utils.py b/cmd2/utils.py index f88a46f20..f9c449346 100644 --- a/cmd2/utils.py +++ b/cmd2/utils.py @@ -690,6 +690,9 @@ def __init__( self.saved_pipe_proc_reader = pipe_proc_reader self.saved_redirecting = saved_redirecting + # Holds the bottom toolbar's suspension while a pipe process owns the terminal + self.toolbar_suspension: contextlib.ExitStack | None = None + def categorize(func: Callable[..., Any] | Iterable[Callable[..., Any]], category: str) -> None: """Categorize a function. diff --git a/docs/features/prompt.md b/docs/features/prompt.md index b1eaed2a2..6a40912f0 100644 --- a/docs/features/prompt.md +++ b/docs/features/prompt.md @@ -115,14 +115,22 @@ fast and use a lock when reading state that a command or another thread modifies ### Commands That Take Over the Terminal cmd2 temporarily hides the toolbar for its input prompts, pagers, Python environments, and shell -commands. It restores the toolbar when those operations finish. For custom terminal UIs, calls to -`input()`, or subprocesses that inherit the terminal, use [cmd2.Cmd.suspend_bottom_toolbar][]: +commands. It also hides it while a command's output is piped to another process, since that process +may be interactive, as `less` and `fzf` are. It restores the toolbar when those operations finish. + +For custom terminal UIs, calls to `input()`, or subprocesses your own command code starts, use +[cmd2.Cmd.suspend_bottom_toolbar][]: ```py with self.suspend_bottom_toolbar(): answer = input("Continue? ") ``` +Suspending matters for subprocesses even when they only print. While the toolbar is displayed, the +terminal is in raw mode, so the kernel generates no signals from keystrokes. cmd2 sends `Ctrl-C` on +to its own process group, which reaches a subprocess your command started and waits on, but `Ctrl-\` +(`SIGQUIT`) does nothing, exactly as at the main prompt. + The command toolbar is used by the interactive command loop, including startup commands and scripts launched from that loop. It is disabled for non-interactive input. Calls to `onecmd_plus_hooks()` outside the command loop do not start a toolbar. diff --git a/examples/getting_started.py b/examples/getting_started.py index 993c45d0c..1c2e3ed83 100755 --- a/examples/getting_started.py +++ b/examples/getting_started.py @@ -38,7 +38,7 @@ Color, stylize, ) -from cmd2.annotated import Option +from cmd2.annotated import Argument, Option class BasicApp(cmd2.Cmd): @@ -166,7 +166,12 @@ def get_rprompt(self) -> AnyFormattedText: return [(style, text)] @cmd2.with_annotated - def do_work(self, seconds: int = 5) -> None: + def do_work( + self, + seconds: Annotated[ # Optional positional argument, so both "work" and "work 5" are valid + int, Argument(nargs="?", help_text="number of seconds to work for") + ] = 5, + ) -> None: """Simulate work while the bottom toolbar clock keeps updating.""" for second in range(seconds): self.poutput(f"Working: {second + 1}/{seconds}") diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index a59322c31..5bdbc3101 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -3,6 +3,7 @@ import io import sys import threading +import time from types import SimpleNamespace from unittest import mock @@ -122,6 +123,46 @@ def test_command_toolbar_pipe_output(toolbar_app) -> None: assert "CMD2 COMMANDS" in output.getvalue() +class FileTerminal: + """A real file that claims to be a terminal, so it owns a descriptor a subprocess can inherit.""" + + def __init__(self, file) -> None: + self.file = file + + def isatty(self) -> bool: + return True + + def __getattr__(self, name): + return getattr(self.file, name) + + +def test_command_toolbar_pipe_process_inherits_terminal(toolbar_app, tmp_path) -> None: + app, _, _ = toolbar_app + destination = tmp_path / "terminal.txt" + running = [] + readers = [] + + def command(statement, **kwargs): + # The pipe process owns the terminal, so the toolbar must have stepped aside. + running.append(app._command_toolbar.app.is_running) + readers.append(app._cur_pipe_proc_reader) + app.poutput("piped") + return False + + with destination.open("w+") as handle: + app.stdout = FileTerminal(handle) + with mock.patch.object(app, "onecmd", side_effect=command), app._command_toolbar_context(): + app.onecmd_plus_hooks(f'custom | "{sys.executable}" -c "import sys; sys.stdout.write(sys.stdin.read().upper())"') + # The terminal goes back to the toolbar once the pipe process has exited. + assert app._command_toolbar.app.is_running + assert app.stdout.proxy is not None + + assert running == [False] + # A process given the terminal writes to it directly instead of through a captured pipe. + assert readers[0]._proc.stdout is None + assert "PIPED" in destination.read_text() + + def test_command_toolbar_binary_output(toolbar_app) -> None: app, _, output = toolbar_app data = "Unicode: πŸ˜‡\n".encode() @@ -135,19 +176,25 @@ def test_command_toolbar_binary_output(toolbar_app) -> None: def test_command_toolbar_interrupt_uses_signal_handler(toolbar_app) -> None: app, pipe, _ = toolbar_app interrupted = threading.Event() - signal_target = "_thread.interrupt_main" if sys.platform == "win32" else "cmd2.command_toolbar.os.kill" + signal_target = "_thread.interrupt_main" if sys.platform == "win32" else "cmd2.command_toolbar.os.killpg" with mock.patch(signal_target, side_effect=lambda *_: interrupted.set()) as interrupt: with app._command_toolbar_context(): pipe.send_text("\x03") assert interrupted.wait(2) interrupt.assert_called_once() + if sys.platform != "win32": + # Reach subprocesses a command started, as the terminal driver would. + import os + import signal + + assert interrupt.call_args.args == (os.getpgrp(), signal.SIGINT) def test_command_toolbar_interrupt_discards_cancelled_typeahead(toolbar_app) -> None: app, pipe, _ = toolbar_app interrupted = threading.Event() received = threading.Event() - signal_target = "_thread.interrupt_main" if sys.platform == "win32" else "cmd2.command_toolbar.os.kill" + signal_target = "_thread.interrupt_main" if sys.platform == "win32" else "cmd2.command_toolbar.os.killpg" with mock.patch(signal_target, side_effect=lambda *_: interrupted.set()), app._command_toolbar_context(): toolbar = app._command_toolbar @@ -226,6 +273,66 @@ def test_command_toolbar_suspension_and_nested_input(toolbar_app) -> None: assert toolbar.app.is_running +class CprOutput(RecordingOutput): + """A terminal that asks for cursor position reports and never answers them.""" + + def get_rows_below_cursor_position(self) -> int: + raise NotImplementedError + + @property + def responds_to_cpr(self) -> bool: + return True + + +def test_command_toolbar_flushes_writes_waiting_on_cursor_reports() -> None: + app = Cmd(allow_cli_args=False) + output = Terminal() + app.stdout = output + + with create_pipe_input() as pipe: + app.main_session = PromptSession( + input=pipe, + output=CprOutput(output), + bottom_toolbar="STATUS", + refresh_interval=0.01, + ) + # Terminal writes wait for a pending cursor position report, so stopping the + # display must not cancel them out from under the text. + with app._command_toolbar_context(): + app.poutput("last words") + + assert "last words\n" in output.getvalue() + + +def test_command_toolbar_suspension_waits_for_in_flight_writes(toolbar_app) -> None: + app, _, output = toolbar_app + writing = threading.Event() + + with app._command_toolbar_context(): + proxy = app._command_toolbar._proxy + proxy_write = proxy.write + + def slow_write(data: str) -> int: + # Widen the window in which suspending could close this proxy. A closed + # proxy accepts writes and discards them, so the output would vanish. + writing.set() + time.sleep(0.1) + return proxy_write(data) + + proxy.write = slow_write + thread = threading.Thread(target=lambda: app.poutput("in flight")) + thread.start() + assert writing.wait(2) + + # A command reaches this at every finalization boundary while its own + # threads are still printing. + with app.suspend_bottom_toolbar(): + pass + thread.join() + + assert "in flight\n" in output.getvalue() + + def test_command_toolbar_typeahead(toolbar_app) -> None: app, pipe, _ = toolbar_app received = threading.Event() @@ -282,6 +389,50 @@ def run_command(): assert app._command_toolbar is None +def test_command_toolbar_recovers_from_stop_failure(toolbar_app) -> None: + app, _, _ = toolbar_app + + context = app._command_toolbar_context() + context.__enter__() + toolbar = app._command_toolbar + real_stop = toolbar.stop + + def failing_stop() -> None: + # Tear down for real, then fail the way a broken stream close would. + real_stop() + raise RuntimeError("broken stop") + + toolbar.stop = failing_stop + with pytest.raises(RuntimeError, match="broken stop"): + context.__exit__(None, None, None) + + # A later command still gets a toolbar instead of being locked out by the dead one. + assert app._command_toolbar is None + with app._command_toolbar_context(): + assert app._command_toolbar is not None + assert app._command_toolbar is not toolbar + assert app._command_toolbar is None + + +def test_command_toolbar_failure_after_startup_is_reported(toolbar_app, capsys) -> None: + app, _, output = toolbar_app + + with app._command_toolbar_context(): + toolbar = app._command_toolbar + # Stop the display the way an unhandled error in its own thread would, after + # _resume() has already returned and can no longer raise for the command. + toolbar.app.loop.call_soon_threadsafe(lambda: toolbar.app.exit(exception=ValueError("broken display"))) + toolbar._thread.join(timeout=2) + assert not toolbar._thread.is_alive() + + # Output must still reach the terminal rather than a proxy nothing is draining. + assert all(stream.proxy is None for stream in toolbar._streams) + app.poutput("after failure") + + assert "broken display" in capsys.readouterr().err + assert "after failure\n" in output.getvalue() + + def test_command_toolbar_render_failure(toolbar_app) -> None: app, _, output = toolbar_app From 170265496e8748bd68042cadd01dc00f3126da79 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sat, 5 Sep 2026 18:48:43 -0400 Subject: [PATCH 03/35] Fix Windows CI failures in command toolbar tests Three tests called _read_raw_input() after the toolbar context had exited, so no prompt-toolkit app session was active. patch_stdout() then asked the default app session for its output, which lazily builds a real terminal Output: harmless Vt100_Output on POSIX, but Win32Output on Windows, which raises NoConsoleScreenBufferError without a console screen buffer of the kind GitHub Actions does not provide. Bind the fixture's app session to the test pipe and recording output so every prompt-toolkit call in these tests resolves to the test terminal rather than the ambient one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C7nkx8kKC2mpaMGfLwXe6J --- tests/test_command_toolbar.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index 5bdbc3101..cb314fc79 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -8,7 +8,7 @@ from unittest import mock import pytest -from prompt_toolkit.application import get_app +from prompt_toolkit.application import create_app_session, get_app from prompt_toolkit.input import create_pipe_input from prompt_toolkit.input.typeahead import get_typeahead from prompt_toolkit.keys import Keys @@ -40,13 +40,18 @@ def toolbar_app(): output = Terminal() app.stdout = output with create_pipe_input() as pipe: - app.main_session = PromptSession( - input=pipe, - output=RecordingOutput(output), - bottom_toolbar="STATUS", - refresh_interval=0.01, - ) - yield app, pipe, output + terminal = RecordingOutput(output) + # Bind the ambient app session to this terminal. Without it, prompt-toolkit + # builds a real one on demand for calls such as patch_stdout() in + # _read_raw_input(), which needs a console that Windows CI does not provide. + with create_app_session(input=pipe, output=terminal): + app.main_session = PromptSession( + input=pipe, + output=terminal, + bottom_toolbar="STATUS", + refresh_interval=0.01, + ) + yield app, pipe, output def test_command_toolbar_refresh_and_output(toolbar_app, monkeypatch) -> None: From aac8a4e7dcac3436172e30e12bf0a036a819ea33 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sat, 5 Sep 2026 18:51:29 -0400 Subject: [PATCH 04/35] Working prototype for one application with interchangeable layouts and an embedded pager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Following prompt-toolkit’s [layout composition model](https://python-prompt-toolkit.readthedocs.io/en/3.0.52/pages/full_screen_apps.html#the-layout). - CommandToolbar (cmd2/command_toolbar.py:101) now reuses main_session.app and its actual toolbar container, eliminating the second application and duplicated display configuration. - The new pager (cmd2/pager.py:37) shares that application. The toolbar continues refreshing during scrolling, searching, and resizing. Colors, wrapped lines, and horizontal scrolling are supported. - Enabling the toolbar selects the embedded pager. Set self.use_builtin_pager = False to retain your external pager. An external pager such as less takes control of the terminal; a layout alone cannot reserve space around it. That path still temporarily suspends the toolbar. This simplifies rendering ownership, although the embedded pager adds code. The command UI thread and output proxy remain necessary for synchronous commands. There’s also one guarded dependency on PromptSession’s internal layout structure, verified against versions 3.0.52 and 3.0.53. --- cmd2/cmd2.py | 29 +++-- cmd2/command_toolbar.py | 166 ++++++++++++++++++++----- cmd2/pager.py | 209 ++++++++++++++++++++++++++++++++ docs/features/initialization.md | 1 + docs/features/os.md | 7 +- docs/features/prompt.md | 16 ++- tests/test_command_toolbar.py | 196 ++++++++++++++++++++++++++++++ 7 files changed, 583 insertions(+), 41 deletions(-) create mode 100644 cmd2/pager.py diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index 432bc9f22..a64bbaa53 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -645,6 +645,10 @@ def __init__( if callargs: self._startup_commands.extend(callargs) + # The embedded pager shares the main toolbar. Applications can opt back + # into their configured external pager by setting this to False. + self.use_builtin_pager = enable_bottom_toolbar + # Set the pager(s) for use when displaying output using a pager if sys.platform.startswith("win"): self.pager = self.pager_chop = "more" @@ -1871,7 +1875,6 @@ def pfeedback( rich_print_kwargs=rich_print_kwargs, ) - @command_toolbar.suspend_toolbar def ppaged( self, *objects: Any, @@ -1893,12 +1896,16 @@ def ppaged( fits on the screen. A pager is not used inside a script (Python or text) or when output is redirected or piped, and in these cases, output is sent to `poutput`. + With the bottom toolbar enabled, the built-in pager keeps it visible and refreshing. + Set ``use_builtin_pager=False`` to use the configured external ``pager`` or ``pager_chop`` + command instead; external pagers temporarily hide the toolbar. + :param chop: True -> causes lines longer than the screen width to be chopped (truncated) rather than wrapped - truncated text is still accessible by scrolling with the right & left arrow keys - chopping is ideal for displaying wide tabular data as is done in utilities like pgcli False -> causes lines longer than the screen width to wrap to the next line - wrapping is ideal when you want to keep users from having to use horizontal scrolling - WARNING: On Windows, the text always wraps regardless of what the chop argument is set to + WARNING: The default external pager on Windows always wraps; the built-in pager supports chopping. :param soft_wrap: Enable soft wrap mode. If True, lines of text will not be word-wrapped or cropped to fit the terminal width. Defaults to True. @@ -1941,11 +1948,19 @@ def ppaged( soft_wrap=soft_wrap, **(rich_print_kwargs if rich_print_kwargs is not None else {}), ) - output_bytes = capture.get().encode("utf-8", "replace") + output = capture.get() + + if self.use_builtin_pager: + with self._command_toolbar_context(): + if self._command_toolbar is not None and self._command_toolbar.is_active: + self._command_toolbar.page(output, chop=chop) + return + + output_bytes = output.encode("utf-8", "replace") # Prevent KeyboardInterrupts while in the pager. The pager application will # still receive the SIGINT since it is in the same process group as us. - with self.sigint_protection: + with self.suspend_bottom_toolbar(), self.sigint_protection: import subprocess pipe_proc = subprocess.Popen( # noqa: S602 @@ -2056,8 +2071,8 @@ def get_bottom_toolbar(self) -> AnyFormattedText: or even a real-time clock. During command execution this callback runs in a background UI thread. Protect shared - state with a lock when necessary. The toolbar is suspended while another prompt, pager, - or interactive shell owns the terminal. + state with a lock when necessary. The built-in pager shares this toolbar. It is suspended + while another prompt, external pager, or interactive shell owns the terminal. :return: Content to populate the bottom toolbar. """ @@ -2069,7 +2084,7 @@ def suspend_bottom_toolbar(self) -> Iterator[None]: Use this context manager around application-specific calls to ``input()``, other terminal UIs, or subprocesses that inherit the terminal. cmd2 automatically suspends - its toolbar for its own input prompts, pagers, and shell commands. + its toolbar for its own input prompts, external pagers, and shell commands. """ if self._command_toolbar is None: yield diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index 90ff014f1..e743d4ee2 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -9,23 +9,28 @@ import sys import threading from collections.abc import Callable, Iterator +from concurrent.futures import Future +from concurrent.futures import TimeoutError as FutureTimeoutError from typing import TYPE_CHECKING, Any, TextIO, TypeVar, cast from prompt_toolkit.application import Application, create_app_session -from prompt_toolkit.filters import Condition, is_done, renderer_height_is_known, to_filter +from prompt_toolkit.enums import EditingMode +from prompt_toolkit.filters import Condition, to_filter from prompt_toolkit.input.typeahead import get_typeahead, store_typeahead from prompt_toolkit.key_binding import KeyBindings from prompt_toolkit.key_binding.key_processor import KeyPress, KeyPressEvent from prompt_toolkit.layout import HSplit, Layout, Window from prompt_toolkit.layout.containers import ConditionalContainer -from prompt_toolkit.layout.controls import FormattedTextControl from prompt_toolkit.patch_stdout import StdoutProxy from prompt_toolkit.utils import suspend_to_background_supported +from .pager import Pager + if TYPE_CHECKING: from .cmd2 import Cmd _F = TypeVar("_F", bound=Callable[..., Any]) +_R = TypeVar("_R") def suspend_toolbar(func: _F) -> _F: @@ -94,7 +99,7 @@ def finish(self) -> None: class CommandToolbar: - """Run a prompt-toolkit display in a thread while a command runs on the main thread. + """Borrow the main prompt's application while a command runs on the main thread. The display owns terminal input so it can receive cursor position reports. Keys typed during execution are saved for the next prompt; Ctrl-C is sent to cmd2's @@ -114,6 +119,23 @@ def __init__(self, cmd: "Cmd") -> None: self._proxy: StdoutProxy | None = None session = cmd.main_session + self.app = session.app + # PromptSession has no public hook for replacing just its input area. + # Keep this small dependency on its layout shape in one place, and fail + # explicitly if upstream changes it. Reuse the actual toolbar container, + # including its visibility filter and support for multiline toolbars. + root = session.layout.container + if not isinstance(root, HSplit): + raise TypeError("Unsupported PromptSession layout") + self.toolbar = root.children[-1] + if not ( + isinstance(self.toolbar, ConditionalContainer) + and isinstance(self.toolbar.content, Window) + and self.toolbar.content.style == "class:bottom-toolbar" + ): + raise RuntimeError("Cannot locate PromptSession bottom toolbar") + self._layout = Layout(HSplit([Window(height=0), Window(), self.toolbar])) + self._display_stack: contextlib.ExitStack | None = None bindings = KeyBindings() @bindings.add("") @@ -143,33 +165,11 @@ def suspend(event: KeyPressEvent) -> None: # redraws the toolbar after the process resumes. event.app.suspend_to_background() - self.app: Application[None] = Application( - layout=Layout( - HSplit( - [ - Window(height=0), - Window(), - ConditionalContainer( - Window( - FormattedTextControl(lambda: session.bottom_toolbar, style="class:bottom-toolbar.text"), - style="class:bottom-toolbar", - height=1, - always_hide_cursor=True, - ), - filter=~is_done & renderer_height_is_known, - ), - ] - ) - ), - input=session.input, - output=session.output, - style=session.style, - color_depth=session.color_depth, - refresh_interval=session.refresh_interval, - key_bindings=bindings, - erase_when_done=True, - after_render=lambda _: self._ready.set(), - ) + self._bindings = bindings + self._suspend_binding = suspend + + def _after_render(self, app: Application[str]) -> None: # noqa: ARG002 + self._ready.set() def start(self) -> None: """Start rendering and protect terminal output.""" @@ -201,6 +201,12 @@ def _restore_stream(obj: Any, name: str, stream: ToolbarStream) -> None: def _resume(self) -> None: self._ready.clear() self._error = None + stack = self._display_stack = contextlib.ExitStack() + for name, value in (("layout", self._layout), ("key_bindings", self._bindings), ("erase_when_done", True)): + stack.callback(setattr, self.app, name, getattr(self.app, name)) + setattr(self.app, name, value) + self.app.after_render += self._after_render + stack.callback(self.app.after_render.remove_handler, self._after_render) context = contextvars.copy_context() def run() -> None: @@ -239,6 +245,9 @@ def _pause(self) -> None: if self._thread is not None: self._thread.join() self._thread = None + if self._display_stack is not None: + self._display_stack.close() + self._display_stack = None # Application.run() saves its unprocessed queue before the thread # exits. Those keys arrived after the ones handled by save_key(). pending_keys = get_typeahead(self.app.input) @@ -256,6 +265,103 @@ def stop(self) -> None: self._stack.close() self._stack = None + @property + def is_active(self) -> bool: + """Whether the display currently owns the terminal.""" + return self._proxy is not None and self.app.is_running + + def _call_in_ui(self, func: Callable[[], _R]) -> _R: + """Change UI state on its event loop, propagating failures to the command.""" + result: Future[_R] = Future() + + def call() -> None: + try: + value = func() + except BaseException as exc: # noqa: BLE001 + result.set_exception(exc) + else: + result.set_result(value) + + if self.app.loop is None: + raise RuntimeError("Toolbar is not running") + self.app.loop.call_soon_threadsafe(call) + while True: + try: + value = result.result(timeout=0.1) + except FutureTimeoutError: + if result.done(): + raise + self._check_running() + else: + return value + + def _check_running(self) -> None: + if self._thread is None or not self._thread.is_alive(): + if self._error is not None: + raise self._error + raise EOFError + + def page(self, text: str, *, chop: bool) -> None: + """Show a pager above the same toolbar without starting another input reader.""" + pager = Pager(text, chop=chop) + pager.bindings.add( + "c-z", + filter=Condition(lambda: suspend_to_background_supported() and to_filter(self.cmd.main_session.enable_suspend)()), + )(self._suspend_binding) + size = self.app.output.get_size() + # Measuring the toolbar can invoke its callback; keep that work on the + # UI thread along with rendering and layout changes. + toolbar_height = self._call_in_ui(lambda: self.toolbar.preferred_height(size.columns, size.rows).preferred) + if pager.fits(size.columns, max(0, size.rows - toolbar_height)): + self.cmd.stdout.write(text) + self.cmd.stdout.flush() + return + + layout = Layout(HSplit([pager.container, self.toolbar]), focused_element=pager.text) + previous = (self.app.layout, self.app.key_bindings, self.app.editing_mode, self.app.full_screen) + entered = False + + def enter() -> None: + nonlocal entered + entered = True + self.app.renderer.erase() + self.app.layout = layout + self.app.key_bindings = pager.bindings + self.app.editing_mode = EditingMode.EMACS + self.app.full_screen = self.app.renderer.full_screen = True + self.app.invalidate() + + def leave() -> None: + nonlocal entered + if not entered: + return + entered = False + self.app.renderer.erase() + self.app.layout, self.app.key_bindings, self.app.editing_mode, self.app.full_screen = previous + self.app.renderer.full_screen = self.app.full_screen + self.app.renderer.request_absolute_cursor_position() + self.app.invalidate() + + def close() -> None: + # Switch bindings before the next key is processed, preserving + # typeahead sent in the same terminal read as the pager's quit key. + leave() + pager.closed.set() + + pager.on_close = close + + try: + self._call_in_ui(enter) + while not pager.closed.wait(0.1): + self._check_running() + finally: + if self._thread is not None and self._thread.is_alive(): + self._call_in_ui(leave) + else: + # The application's shutdown already reset the renderer. + self.app.layout, self.app.key_bindings, self.app.editing_mode, self.app.full_screen = previous + self.app.renderer.full_screen = self.app.full_screen + @contextlib.contextmanager def suspend(self) -> Iterator[None]: """Temporarily restore ordinary terminal access, including nested suspensions.""" diff --git a/cmd2/pager.py b/cmd2/pager.py new file mode 100644 index 000000000..9d183b20d --- /dev/null +++ b/cmd2/pager.py @@ -0,0 +1,209 @@ +"""A pager view hosted by the main prompt-toolkit application.""" + +import threading +from collections.abc import Callable +from functools import partial + +from prompt_toolkit.document import Document +from prompt_toolkit.filters import has_focus, is_searching, to_filter +from prompt_toolkit.formatted_text import ANSI, fragment_list_to_text, to_formatted_text +from prompt_toolkit.formatted_text.base import StyleAndTextTuples +from prompt_toolkit.formatted_text.utils import split_lines +from prompt_toolkit.key_binding import KeyBindings +from prompt_toolkit.key_binding.bindings import search +from prompt_toolkit.key_binding.key_processor import KeyPressEvent +from prompt_toolkit.layout import HSplit, Window +from prompt_toolkit.layout.containers import ConditionalContainer +from prompt_toolkit.layout.controls import FormattedTextControl, UIContent +from prompt_toolkit.lexers import Lexer +from prompt_toolkit.search import SearchDirection, start_search +from prompt_toolkit.utils import get_cwidth +from prompt_toolkit.widgets import SearchToolbar, TextArea + + +class _AnsiLexer(Lexer): + """Preserve captured Rich styles while searching and scrolling plain text.""" + + def __init__(self, fragments: StyleAndTextTuples) -> None: + self.lines = list(split_lines(fragments)) + + def lex_document(self, document: Document) -> Callable[[int], StyleAndTextTuples]: # noqa: ARG002 + def get_line(number: int) -> StyleAndTextTuples: + return self.lines[number] if number < len(self.lines) else [] + + return get_line + + +class Pager: + """Scrollable, searchable output; the host supplies the persistent toolbar.""" + + def __init__(self, text: str, *, chop: bool) -> None: + """Build an independent view without creating an application or input reader.""" + self.closed = threading.Event() + self.on_close: Callable[[], None] = self.closed.set + self.chop = chop + fragments = to_formatted_text(ANSI(text.removesuffix("\n"))) + self.search = SearchToolbar() + self.text = TextArea( + text=fragment_list_to_text(fragments), + lexer=_AnsiLexer(fragments), + read_only=True, + wrap_lines=not chop, + search_field=self.search, + ) + self.text.window.always_hide_cursor = to_filter(True) + self.container = HSplit( + [ + self.text, + self.search, + ConditionalContainer( + Window( + FormattedTextControl(" Space/PgDn: next b/PgUp: back /: search n: next match q: quit"), + height=1, + style="class:bottom-toolbar", + ), + filter=~is_searching, + ), + ] + ) + self.bindings = bindings = KeyBindings() + focused = has_focus(self.text) + + @bindings.add("", filter=focused) + def ignore(event: KeyPressEvent) -> None: + # Pager keystrokes must not become commands at the next prompt. + pass + + @bindings.add("q", filter=focused) + @bindings.add("escape", filter=focused, eager=True) + @bindings.add("c-c", filter=focused) + def close(event: KeyPressEvent) -> None: # noqa: ARG001 + self.on_close() + + for keys, pages in ( + ((" ", "pagedown", "f", "c-f"), 1.0), + (("b", "pageup", "c-b"), -1.0), + (("d", "c-d"), 0.5), + (("u", "c-u"), -0.5), + ): + for key in keys: + bindings.add(key, filter=focused)(partial(self._scroll_page, pages=pages)) + + @bindings.add("j", filter=focused) + @bindings.add("down", filter=focused) + @bindings.add("enter", filter=focused) + def down(event: KeyPressEvent) -> None: + self._scroll(event, 1) + + @bindings.add("k", filter=focused) + @bindings.add("up", filter=focused) + def up(event: KeyPressEvent) -> None: + self._scroll(event, -1) + + @bindings.add("right", filter=focused) + @bindings.add("l", filter=focused) + def right(event: KeyPressEvent) -> None: + self._scroll_horizontal(event, 1) + + @bindings.add("left", filter=focused) + @bindings.add("h", filter=focused) + def left(event: KeyPressEvent) -> None: + self._scroll_horizontal(event, -1) + + @bindings.add("g", filter=focused) + @bindings.add("home", filter=focused) + def first(event: KeyPressEvent) -> None: + event.current_buffer.cursor_position = 0 + + @bindings.add("G", filter=focused) + @bindings.add("end", filter=focused) + def last(event: KeyPressEvent) -> None: + event.current_buffer.cursor_position = len(event.current_buffer.text) + + @bindings.add("/", filter=focused) + def find(event: KeyPressEvent) -> None: # noqa: ARG001 + start_search(direction=SearchDirection.FORWARD) + + @bindings.add("?", filter=focused) + def find_backwards(event: KeyPressEvent) -> None: # noqa: ARG001 + start_search(direction=SearchDirection.BACKWARD) + + @bindings.add("n", filter=focused) + def next_match(event: KeyPressEvent) -> None: + event.current_buffer.apply_search(event.app.current_search_state, include_current_position=False) + + @bindings.add("N", filter=focused) + def previous_match(event: KeyPressEvent) -> None: + event.current_buffer.apply_search(~event.app.current_search_state, include_current_position=False) + + # Explicit search bindings also work when the main prompt uses Vi mode. + bindings.add("enter", filter=is_searching)(search.accept_search) + bindings.add("escape", filter=is_searching, eager=True)(search.abort_search) + bindings.add("c-c", filter=is_searching)(search.abort_search) + + @staticmethod + def _column_at_width(line: str, width: int) -> int: + used = 0 + for column, char in enumerate(line): + if used >= width: + return column + used += get_cwidth(char) + return len(line) + + def _scroll_page(self, event: KeyPressEvent, *, pages: float) -> None: + info = self.text.window.render_info + if info is not None: + amount = max(1, int(max(1, info.window_height - 1) * abs(pages))) + self._scroll(event, amount if pages > 0 else -amount) + + def _scroll(self, event: KeyPressEvent, rows: int) -> None: + """Move by display rows, including within lines taller than the viewport.""" + info = self.text.window.render_info + if info is None or info.window_width == 0: + return + document = event.current_buffer.document + line = document.cursor_position_row + wrapped_row = 0 if self.chop else get_cwidth(document.current_line_before_cursor) // info.window_width + target = wrapped_row + rows + + def height(number: int) -> int: + return 1 if self.chop else info.get_height_for_line(number) + + while target < 0 and line > 0: + line -= 1 + target += height(line) + while target >= height(line) and line < document.line_count - 1: + target -= height(line) + line += 1 + target = max(0, min(target, height(line) - 1)) + column = self._column_at_width(document.lines[line], target * info.window_width) + event.current_buffer.cursor_position = document.translate_row_col_to_index(line, column) + self.text.window.vertical_scroll = line + self.text.window.vertical_scroll_2 = target if height(line) > info.window_height else 0 + + def _scroll_horizontal(self, event: KeyPressEvent, direction: int) -> None: + info = self.text.window.render_info + if info is None or not self.chop: + return + document = event.current_buffer.document + target = max(0, self.text.window.horizontal_scroll + direction * max(1, info.window_width // 2)) + column = self._column_at_width(document.current_line, target) + event.current_buffer.cursor_position = document.translate_row_col_to_index(document.cursor_position_row, column) + self.text.window.horizontal_scroll = get_cwidth(document.current_line[:column]) + + def fits(self, columns: int, rows: int) -> bool: + """Check rendered line heights, including wrapping and wide Unicode characters.""" + # Measuring must not ask BufferControl to create content: that starts + # history-loading tasks and is only safe on the application's event loop. + lines = self.text.document.lines + if self.chop: + # Even one wide line needs a pager so its hidden columns remain + # accessible through horizontal scrolling. + return len(lines) <= rows and all(get_cwidth(line) <= columns for line in lines) + content = UIContent(get_line=lambda number: [("", lines[number])], line_count=len(lines)) + height = 0 + for line in range(content.line_count): + height += content.get_height_for_line(line, columns, None) + if height > rows: + return False + return True diff --git a/docs/features/initialization.md b/docs/features/initialization.md index fc084ff3d..74b463174 100644 --- a/docs/features/initialization.md +++ b/docs/features/initialization.md @@ -52,6 +52,7 @@ Here are instance attributes of `cmd2.Cmd` which developers might wish to overri - **max_completion_table_items**: The maximum number of completion results allowed for a completion table to appear (Default: 50) - **pager**: sets the pager command used by the `Cmd.ppaged()` method for displaying wrapped output using a pager - **pager_chop**: sets the pager command used by the `Cmd.ppaged()` method for displaying chopped/truncated output using a pager +- **use_builtin_pager**: defaults to `enable_bottom_toolbar`; when enabled, `Cmd.ppaged()` uses an embedded pager with the persistent toolbar. Set to `False` to use the external `pager`/`pager_chop` commands. - **py_bridge_name**: name by which embedded Python environments and scripts refer to the `cmd2` application by in order to call commands (Default: `app`) - **py_locals**: dictionary that defines specific variables/functions available in Python shells and scripts (provides more fine-grained control than making everything available with **self_in_py**) - **quiet**: if `True`, then completely suppress nonessential output (Default: `False`) diff --git a/docs/features/os.md b/docs/features/os.md index 444822fd6..22a495c81 100644 --- a/docs/features/os.md +++ b/docs/features/os.md @@ -44,6 +44,11 @@ system. Output of any command can be displayed one page at a time using the [cmd2.Cmd.ppaged][] method. +When the bottom toolbar is enabled, `ppaged()` uses an embedded pager so the toolbar remains visible +and continues refreshing. It supports scrolling, search, and chopped lines. Set +`self.use_builtin_pager = False` to use the configured external `pager`/`pager_chop` commands +instead. See [Bottom Toolbar](./prompt.md#bottom-toolbar) for controls and details. + Alternatively, a terminal pager can be invoked directly using the ability to run shell commands with the `!` shortcut like so: @@ -51,7 +56,7 @@ the `!` shortcut like so: !!! warning - Once you are in a terminal pager, that program temporarily has control of your terminal, + Once you are in an external terminal pager, that program temporarily has control of your terminal, **NOT** `cmd2`. Typically you can use either the arrow keys or ``/`` keys to scroll around or type `q` to quit the pager and return control to your `cmd2` application. diff --git a/docs/features/prompt.md b/docs/features/prompt.md index b1eaed2a2..c92c6f52b 100644 --- a/docs/features/prompt.md +++ b/docs/features/prompt.md @@ -114,9 +114,19 @@ fast and use a lock when reading state that a command or another thread modifies ### Commands That Take Over the Terminal -cmd2 temporarily hides the toolbar for its input prompts, pagers, Python environments, and shell -commands. It restores the toolbar when those operations finish. For custom terminal UIs, calls to -`input()`, or subprocesses that inherit the terminal, use [cmd2.Cmd.suspend_bottom_toolbar][]: +With the toolbar enabled, `ppaged()` uses an embedded pager that keeps the same toolbar visible and +refreshing. Use Space or PageDown to advance, `b` or PageUp to go back, `/` to search, `n`/`N` to +repeat a search, and `q` to return to the command. Arrow keys navigate the output; `g`/`G` jump to +the beginning/end. Short output is printed directly above the toolbar. Chopped output supports +horizontal scrolling, including on Windows. + +Set `self.use_builtin_pager = False` to use your configured external `pager`/`pager_chop` commands. +The embedded pager provides basic navigation and search; applications that need additional `less` +features can use this opt-out. + +cmd2 temporarily hides the toolbar for its input prompts, external pagers, Python environments, and +shell commands. It restores the toolbar when those operations finish. For custom terminal UIs, calls +to `input()`, or subprocesses that inherit the terminal, use [cmd2.Cmd.suspend_bottom_toolbar][]: ```py with self.suspend_bottom_toolbar(): diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index a59322c31..f25b3a23b 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -3,11 +3,13 @@ import io import sys import threading +from concurrent.futures import ThreadPoolExecutor from types import SimpleNamespace from unittest import mock import pytest from prompt_toolkit.application import get_app +from prompt_toolkit.data_structures import Size from prompt_toolkit.input import create_pipe_input from prompt_toolkit.input.typeahead import get_typeahead from prompt_toolkit.keys import Keys @@ -15,6 +17,7 @@ from prompt_toolkit.shortcuts import PromptSession from cmd2 import Cmd +from cmd2.pager import Pager class Terminal(io.StringIO): @@ -25,6 +28,10 @@ def isatty(self) -> bool: class RecordingOutput(DummyOutput): def __init__(self, stream: Terminal) -> None: self.stdout = stream + self.size = Size(rows=24, columns=80) + + def get_size(self) -> Size: + return self.size def write(self, data: str) -> None: self.stdout.write(data) @@ -260,6 +267,7 @@ def exit_after_first_key(_): assert exiting.wait(2) toolbar._thread.join(timeout=2) assert not toolbar._thread.is_alive() + toolbar.app.key_processor.after_key_press -= exit_after_first_key assert app._read_raw_input("Next: ", app.main_session) == "ab" @@ -317,3 +325,191 @@ def command(line, **kwargs): monkeypatch.setattr(app, "onecmd_plus_hooks", command) app._cmdloop() assert commands == ["startup", "quit"] + + +def test_command_toolbar_reuses_prompt_application(toolbar_app) -> None: + app, pipe, _ = toolbar_app + session = app.main_session + layout, bindings, erase = session.app.layout, session.app.key_bindings, session.app.erase_when_done + with app._command_toolbar_context(): + toolbar = app._command_toolbar + assert toolbar.app is session.app + assert toolbar.toolbar is session.layout.container.children[-1] + with app.suspend_bottom_toolbar(): + assert session.app.layout is layout + assert session.app.key_bindings is bindings + assert session.app.erase_when_done is erase + assert session.app.layout is toolbar._layout + assert session.app.layout is layout + assert session.app.key_bindings is bindings + assert session.app.erase_when_done is erase + assert app._read_raw_input("Next: ", session, pre_run=lambda: pipe.send_text("answer\n")) == "answer" + + +@pytest.mark.parametrize("quit_key", ["q", "\x03"]) +def test_builtin_pager_keeps_toolbar_live(toolbar_app, monkeypatch, quit_key) -> None: + app, pipe, _ = toolbar_app + app.use_builtin_pager = True + monkeypatch.setattr(app, "stdin", Terminal()) + monkeypatch.setenv("TERM", "xterm") + entered, refreshed, moved, found = (threading.Event() for _ in range(4)) + state = ["BEFORE"] + + def toolbar_text(): + assert threading.current_thread() is not threading.main_thread() + if app.main_session.app.full_screen and state[0] == "AFTER": + refreshed.set() + return state[0] + + app.main_session.bottom_toolbar = toolbar_text + prompt_layout = app.main_session.app.layout + + def observe(ui): + if not ui.full_screen: + return + assert ui.layout.container.children[-1] is app._command_toolbar.toolbar + entered.set() + row = ui.layout.current_buffer.document.cursor_position_row + if row > 0: + moved.set() + if row == 80: + found.set() + + def interact(): + try: + assert entered.wait(2) + state[0] = "AFTER" + assert refreshed.wait(2) + pipe.send_text(" ") + assert moved.wait(2) + pipe.send_text("/row 080\n") + assert found.wait(2) + finally: + pipe.send_text(quit_key) + + app.main_session.app.after_render += observe + with mock.patch("subprocess.Popen") as external, ThreadPoolExecutor() as executor: + interaction = executor.submit(interact) + with app._command_toolbar_context(): + thread = app._command_toolbar._thread + app.ppaged("\n".join(f"row {i:03d}" for i in range(100))) + assert app._command_toolbar._thread is thread + assert app._command_toolbar.is_active + assert not app.main_session.app.full_screen + assert not app.main_session.app.renderer.full_screen + interaction.result(timeout=2) + external.assert_not_called() + assert app.main_session.app.layout is prompt_layout + assert refreshed.is_set() + assert get_typeahead(pipe) == [] + + +def test_builtin_pager_short_output(toolbar_app, monkeypatch) -> None: + app, _, output = toolbar_app + app.use_builtin_pager = True + monkeypatch.setattr(app, "stdin", Terminal()) + monkeypatch.setenv("TERM", "xterm") + with mock.patch("subprocess.Popen") as external, app._command_toolbar_context(): + app.ppaged("short output") + assert app._command_toolbar.is_active + external.assert_not_called() + assert "short output\n" in output.getvalue() + + +def test_external_pager_suspends_shared_application(toolbar_app, monkeypatch) -> None: + app, _, _ = toolbar_app + app.use_builtin_pager = False + monkeypatch.setattr(app, "stdin", Terminal()) + monkeypatch.setenv("TERM", "xterm") + + def external(*args, **kwargs): + assert not app.main_session.app.is_running + assert app.main_session.app.layout is app.main_session.layout + return mock.Mock() + + with mock.patch("subprocess.Popen", side_effect=external), app._command_toolbar_context(): + app.ppaged("external pager") + assert app._command_toolbar.is_active + + +def test_builtin_pager_eof_restores_prompt(toolbar_app) -> None: + app, pipe, output = toolbar_app + layout = app.main_session.app.layout + + def close_input(ui): + if ui.full_screen: + pipe.close() + + app.main_session.app.after_render += close_input + with pytest.raises(EOFError), app._command_toolbar_context(): + app._command_toolbar.page("line\n" * 100, chop=False) + assert app.main_session.app.layout is layout + assert not app.main_session.app.full_screen + assert not app.main_session.app.renderer.full_screen + assert app.stdout is output + + +@pytest.mark.parametrize("chop", [False, True]) +def test_pager_styles_and_wrapping(chop) -> None: + pager = Pager("\x1b[31m" + "η•Œ" * 40 + "\x1b[0m\n", chop=chop) + assert pager.text.text == "η•Œ" * 40 + assert pager.text.read_only + lexer = pager.text.lexer.lex_document(pager.text.document) + assert "ansired" in lexer(0)[0][0] + assert not pager.fits(20, 1) + assert pager.fits(20, 5) is (not chop) + assert pager.fits(100, 1) + + +@pytest.mark.parametrize("chop", [False, True]) +def test_pager_long_line_navigation_resize_and_typeahead(toolbar_app, chop) -> None: + app, pipe, _ = toolbar_app + app.main_session.bottom_toolbar = "STATUS ONE\nSTATUS TWO" + entered, scrolled, resized = (threading.Event() for _ in range(3)) + + def observe(ui): + if not ui.full_screen: + return + # Check the rendered frame, not the stream of incremental terminal + # writes, to verify both toolbar rows survive navigation and resizing. + screen = ui.renderer._last_screen + size = ui.output.get_size() + bottom = "".join(screen.data_buffer[size.rows - 1][x].char for x in range(size.columns)) + assert bottom.startswith("STATUS TWO") + entered.set() + if ui.current_buffer.cursor_position > 0: + scrolled.set() + if size.columns == 60: + resized.set() + + def interact(): + try: + assert entered.wait(2) + pipe.send_text("\x1b[C" if chop else " ") + assert scrolled.wait(2) + app.main_session.output.size = Size(rows=20, columns=60) + app.main_session.app.invalidate() + assert resized.wait(2) + finally: + pipe.send_text("qnext\n") + + app.main_session.app.after_render += observe + with ThreadPoolExecutor() as executor: + interaction = executor.submit(interact) + with app._command_toolbar_context(): + app._command_toolbar.page("η•Œ" * 4000, chop=chop) + interaction.result(timeout=2) + assert app._read_raw_input("Next: ", app.main_session) == "next" + + +def test_builtin_pager_does_not_capture_redirected_output(toolbar_app, monkeypatch, tmp_path) -> None: + app, _, output = toolbar_app + app.use_builtin_pager = True + monkeypatch.setattr(app, "stdin", Terminal()) + monkeypatch.setenv("TERM", "xterm") + target = tmp_path / "help.txt" + with mock.patch("cmd2.command_toolbar.Pager") as pager, app._command_toolbar_context(): + app.onecmd_plus_hooks(f'help > "{target}"') + pager.assert_not_called() + assert "Cmd2 Commands" in target.read_text() + assert "Cmd2 Commands" not in output.getvalue() From 6cb419d9748794af8ae0646235bad4a433ee3135 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sat, 5 Sep 2026 19:10:26 -0400 Subject: [PATCH 05/35] Cover the toolbar's EOF shutdown path Closing the terminal's input ends the display's Application.run() with EOFError, which _resume() swallows because it is an ordinary shutdown rather than a failure the running command should hear about. Nothing exercised that handler, leaving it as the one uncovered line in the new module. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C7nkx8kKC2mpaMGfLwXe6J --- tests/test_command_toolbar.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index cb314fc79..c3c8a99d8 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -438,6 +438,28 @@ def test_command_toolbar_failure_after_startup_is_reported(toolbar_app, capsys) assert "after failure\n" in output.getvalue() +def test_command_toolbar_input_eof_is_not_reported(toolbar_app, capsys) -> None: + app, pipe, output = toolbar_app + + with app._command_toolbar_context(): + toolbar = app._command_toolbar + # Losing the terminal's input ends the display with EOFError. That is an + # ordinary shutdown, not a failure the running command should hear about. + pipe.close() + toolbar._thread.join(timeout=2) + assert not toolbar._thread.is_alive() + assert toolbar._error is None + + # Output must still reach the terminal rather than a proxy nothing is draining. + assert all(stream.proxy is None for stream in toolbar._streams) + app.poutput("after eof") + + assert capsys.readouterr().err == "" + assert "after eof\n" in output.getvalue() + assert app.stdout is output + assert app._command_toolbar is None + + def test_command_toolbar_render_failure(toolbar_app) -> None: app, _, output = toolbar_app From 7340427b7d3d7b85c9b9972339ad6ad93c71c7df Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sat, 5 Sep 2026 19:49:45 -0400 Subject: [PATCH 06/35] Added unit tests to completely cover code added for command_toolbar.py and pager.py --- tests/test_command_toolbar.py | 179 +++++++++++++++++++++++++++++++++- 1 file changed, 178 insertions(+), 1 deletion(-) diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index 84877d5b5..0e1df32b1 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -14,6 +14,7 @@ from prompt_toolkit.input import create_pipe_input from prompt_toolkit.input.typeahead import get_typeahead from prompt_toolkit.keys import Keys +from prompt_toolkit.layout import HSplit, Layout, Window from prompt_toolkit.output import DummyOutput from prompt_toolkit.shortcuts import PromptSession @@ -449,6 +450,68 @@ def already_exiting(): assert app.main_session.app.layout is app.main_session.layout +def test_command_toolbar_ui_call_propagates_failures(toolbar_app) -> None: + app, _, _ = toolbar_app + + def fail(exception: BaseException) -> None: + raise exception + + with app._command_toolbar_context(): + toolbar = app._command_toolbar + + # A UI callback runs on the display's loop, so its failure has to be carried + # back to the command thread rather than reaching the loop's error handler. + with pytest.raises(ValueError, match="broken ui call"): + toolbar._call_in_ui(lambda: fail(ValueError("broken ui call"))) + + # A TimeoutError raised by the callback is the same class the pending future + # reports itself with, and must not be mistaken for one. + with pytest.raises(TimeoutError, match="slow ui call"): + toolbar._call_in_ui(lambda: fail(TimeoutError("slow ui call"))) + + # A callback that outlives the poll interval keeps waiting instead of giving up. + assert toolbar._call_in_ui(lambda: time.sleep(0.2) or "finished") == "finished" + + +def test_command_toolbar_ui_call_after_display_stopped(toolbar_app) -> None: + app, pipe, _ = toolbar_app + with app._command_toolbar_context(): + toolbar = app._command_toolbar + pipe.close() + toolbar._thread.join(timeout=2) + assert not toolbar._thread.is_alive() + + # There is no loop left to run UI work on, so asking must fail rather than + # queue a callback onto a closed loop. + assert toolbar.app.loop is None + with pytest.raises(RuntimeError, match="Toolbar is not running"): + toolbar._call_in_ui(lambda: None) + + +def test_command_toolbar_ui_call_reports_display_failure(toolbar_app, capsys) -> None: + app, _, _ = toolbar_app + with app._command_toolbar_context(): + toolbar = app._command_toolbar + loop = toolbar.app.loop + schedule = loop.call_soon_threadsafe + failed = threading.Event() + + def die(*_args, **_kwargs) -> None: + # The display dies instead of running the queued callback, so the future + # the command is waiting on never resolves. + if not failed.is_set(): + failed.set() + schedule(lambda: toolbar.app.exit(exception=ValueError("broken display"))) + + with ( + mock.patch.object(loop, "call_soon_threadsafe", side_effect=die), + pytest.raises(ValueError, match="broken display"), + ): + toolbar._call_in_ui(lambda: None) + + assert "broken display" in capsys.readouterr().err + + def test_command_toolbar_failure_after_startup_is_reported(toolbar_app, capsys) -> None: app, _, output = toolbar_app @@ -498,7 +561,7 @@ def broken_toolbar(): app.main_session.bottom_toolbar = broken_toolbar with pytest.raises(ValueError, match="broken toolbar"), app._command_toolbar_context(): - pytest.fail("Command should not run after a toolbar startup failure") + pytest.fail("Command should not run after a toolbar startup failure") # pragma: no cover assert app.stdout is output assert app._command_toolbar is None @@ -527,6 +590,23 @@ def command(line, **kwargs): assert commands == ["startup", "quit"] +@pytest.mark.parametrize( + ("layout", "error", "message"), + [ + (Layout(Window()), TypeError, "Unsupported PromptSession layout"), + (Layout(HSplit([Window()])), RuntimeError, "Cannot locate PromptSession bottom toolbar"), + ], +) +def test_command_toolbar_requires_the_prompt_toolbar(toolbar_app, layout, error, message) -> None: + app, _, output = toolbar_app + # The display reuses the prompt's own toolbar container. If a future prompt-toolkit + # release moves it, say so instead of rendering something wrong. + with mock.patch.object(app.main_session, "layout", layout), pytest.raises(error, match=message): + app._command_toolbar_context().__enter__() + assert app._command_toolbar is None + assert app.stdout is output + + def test_command_toolbar_reuses_prompt_application(toolbar_app) -> None: app, pipe, _ = toolbar_app session = app.main_session @@ -713,3 +793,100 @@ def test_builtin_pager_does_not_capture_redirected_output(toolbar_app, monkeypat pager.assert_not_called() assert "Cmd2 Commands" in target.read_text() assert "Cmd2 Commands" not in output.getvalue() + + +class PagerKeys: + """Send keys to the built-in pager and wait for each one to be handled.""" + + def __init__(self, app, pipe, pager) -> None: + self.pipe = pipe + self.pager = pager + self.states = [] + self.updated = threading.Condition() + app.main_session.app.key_processor.after_key_press += self._record + + def _record(self, _) -> None: + # Read the pager's own buffer rather than the focused one, which is the + # search field while a search is being typed. + with self.updated: + self.states.append((self.pager.text.buffer.document.cursor_position_row, self.pager.text.window.horizontal_scroll)) + self.updated.notify_all() + + def press(self, keys, row, column=0) -> None: + """Send keys and wait until a resulting position matches, so steps stay ordered.""" + with self.updated: + index = len(self.states) + self.pipe.send_text(keys) + deadline = time.monotonic() + 5 + with self.updated: + while True: + while index < len(self.states): + state = self.states[index] + index += 1 + if state == (row, column): + return + remaining = deadline - time.monotonic() + notified = remaining > 0 and self.updated.wait(remaining) + assert notified, f"pager ignored {keys!r}: wanted {(row, column)}, saw {self.states}" + + +@pytest.mark.parametrize("chop", [False, True]) +def test_pager_navigation_keys(toolbar_app, chop) -> None: + app, pipe, _ = toolbar_app + lines = [f"row {index:03d}" for index in range(100)] + lines[3] = "" # A line with no columns for a scroll target to be clamped against. + created = [] + entered = threading.Event() + + def make_pager(*args, **kwargs): + pager = Pager(*args, **kwargs) + created.append(pager) + return pager + + def observe(ui): + if created and ui.full_screen and ui.layout.current_buffer is created[0].text.buffer: + entered.set() + + def interact(): + try: + assert entered.wait(5) + keys = PagerKeys(app, pipe, created[0]) + keys.press("j", row=1) + keys.press("j", row=2) + keys.press("j", row=3) # Land on the empty line. + keys.press("k", row=2) # Moving up off it re-enters the line above. + keys.press("G", row=99) + keys.press("g", row=0) + keys.press("/row 05\n", row=50) + keys.press("n", row=51) + keys.press("N", row=50) + keys.press("?row 01\n", row=19) + keys.press("x", row=19) # Unbound keys are swallowed, not queued for the prompt. + # Horizontal scrolling applies only to chopped output, and stops at the + # end of a line shorter than the requested column. + keys.press("\x1b[C", row=19, column=len("row 019") if chop else 0) + keys.press("\x1b[D", row=19, column=0) + finally: + pipe.send_text("q") + + app.main_session.app.after_render += observe + with mock.patch("cmd2.command_toolbar.Pager", side_effect=make_pager), ThreadPoolExecutor() as executor: + interaction = executor.submit(interact) + with app._command_toolbar_context(): + app._command_toolbar.page("\n".join(lines), chop=chop) + interaction.result(timeout=10) + assert get_typeahead(pipe) == [] + + +@pytest.mark.parametrize("chop", [False, True]) +def test_pager_scrolling_before_first_render(chop) -> None: + pager = Pager("row\n" * 100, chop=chop) + # Keys can arrive before the first frame is drawn, when the window still has no + # rendered geometry to scroll against. + assert pager.text.window.render_info is None + event = SimpleNamespace(current_buffer=pager.text.buffer) + pager._scroll_page(event, pages=1.0) + pager._scroll(event, 1) + pager._scroll_horizontal(event, 1) + assert pager.text.buffer.cursor_position == 0 + assert pager.text.window.horizontal_scroll == 0 From 4915ea757d32f500f44284d92ec18c2adf01cda8 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sat, 5 Sep 2026 23:02:29 -0400 Subject: [PATCH 07/35] Measure paged output before building a pager page() constructed a full Pager -- a TextArea, a SearchToolbar, an ANSI lexer and its key bindings -- and then threw it away whenever the output fit on screen, which is a common case for ppaged(). Only the line measurements were ever used on that path. Move the size check into a module-level output_fits() that measures the text directly, and build the Pager only once the output has to be scrolled. A shared _fragments() helper does the one ANSI parse for both, so the measurement cannot drift from what the view would render. The fast path's size check drops from ~0.15ms to ~0.02ms per call, so it isn't a massive speedup or anything, but still cleaner. --- cmd2/command_toolbar.py | 14 +++++------ cmd2/pager.py | 46 +++++++++++++++++++++-------------- tests/test_command_toolbar.py | 28 ++++++++++++++++++--- 3 files changed, 59 insertions(+), 29 deletions(-) diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index 68ac6e751..da8c1c881 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -24,7 +24,7 @@ from prompt_toolkit.patch_stdout import StdoutProxy from prompt_toolkit.utils import suspend_to_background_supported -from .pager import Pager +from .pager import Pager, output_fits if TYPE_CHECKING: from .cmd2 import Cmd @@ -389,20 +389,20 @@ def _check_running(self) -> None: def page(self, text: str, *, chop: bool) -> None: """Show a pager above the same toolbar without starting another input reader.""" - pager = Pager(text, chop=chop) - pager.bindings.add( - "c-z", - filter=Condition(lambda: suspend_to_background_supported() and to_filter(self.cmd.main_session.enable_suspend)()), - )(self._suspend_binding) size = self.app.output.get_size() # Measuring the toolbar can invoke its callback; keep that work on the # UI thread along with rendering and layout changes. toolbar_height = self._call_in_ui(lambda: self.toolbar.preferred_height(size.columns, size.rows).preferred) - if pager.fits(size.columns, max(0, size.rows - toolbar_height)): + if output_fits(text, size.columns, max(0, size.rows - toolbar_height), chop=chop): self.cmd.stdout.write(text) self.cmd.stdout.flush() return + pager = Pager(text, chop=chop) + pager.bindings.add( + "c-z", + filter=Condition(lambda: suspend_to_background_supported() and to_filter(self.cmd.main_session.enable_suspend)()), + )(self._suspend_binding) layout = Layout(HSplit([pager.container, self.toolbar]), focused_element=pager.text) previous = (self.app.layout, self.app.key_bindings, self.app.editing_mode, self.app.full_screen) entered = False diff --git a/cmd2/pager.py b/cmd2/pager.py index 9d183b20d..97b9ec41e 100644 --- a/cmd2/pager.py +++ b/cmd2/pager.py @@ -21,6 +21,33 @@ from prompt_toolkit.widgets import SearchToolbar, TextArea +def _fragments(text: str) -> StyleAndTextTuples: + """Parse captured output, dropping the trailing newline that ends its last line.""" + return to_formatted_text(ANSI(text.removesuffix("\n"))) + + +def output_fits(text: str, columns: int, rows: int, *, chop: bool) -> bool: + """Check rendered line heights, including wrapping and wide Unicode characters. + + Measuring the text itself keeps output that needs no scrolling from paying for a + Pager's widgets and key bindings, which would only be built to be thrown away. + """ + lines = fragment_list_to_text(_fragments(text)).split("\n") + if chop: + # Even one wide line needs a pager so its hidden columns remain + # accessible through horizontal scrolling. + return len(lines) <= rows and all(get_cwidth(line) <= columns for line in lines) + # Reuse prompt-toolkit's own wrapping arithmetic so this matches what a Pager + # would render, without building a control to ask on the UI thread. + content = UIContent(get_line=lambda number: [("", lines[number])], line_count=len(lines)) + height = 0 + for line in range(content.line_count): + height += content.get_height_for_line(line, columns, None) + if height > rows: + return False + return True + + class _AnsiLexer(Lexer): """Preserve captured Rich styles while searching and scrolling plain text.""" @@ -42,7 +69,7 @@ def __init__(self, text: str, *, chop: bool) -> None: self.closed = threading.Event() self.on_close: Callable[[], None] = self.closed.set self.chop = chop - fragments = to_formatted_text(ANSI(text.removesuffix("\n"))) + fragments = _fragments(text) self.search = SearchToolbar() self.text = TextArea( text=fragment_list_to_text(fragments), @@ -190,20 +217,3 @@ def _scroll_horizontal(self, event: KeyPressEvent, direction: int) -> None: column = self._column_at_width(document.current_line, target) event.current_buffer.cursor_position = document.translate_row_col_to_index(document.cursor_position_row, column) self.text.window.horizontal_scroll = get_cwidth(document.current_line[:column]) - - def fits(self, columns: int, rows: int) -> bool: - """Check rendered line heights, including wrapping and wide Unicode characters.""" - # Measuring must not ask BufferControl to create content: that starts - # history-loading tasks and is only safe on the application's event loop. - lines = self.text.document.lines - if self.chop: - # Even one wide line needs a pager so its hidden columns remain - # accessible through horizontal scrolling. - return len(lines) <= rows and all(get_cwidth(line) <= columns for line in lines) - content = UIContent(get_line=lambda number: [("", lines[number])], line_count=len(lines)) - height = 0 - for line in range(content.line_count): - height += content.get_height_for_line(line, columns, None) - if height > rows: - return False - return True diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index 0e1df32b1..7820deb64 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -19,7 +19,7 @@ from prompt_toolkit.shortcuts import PromptSession from cmd2 import Cmd -from cmd2.pager import Pager +from cmd2.pager import Pager, output_fits class Terminal(io.StringIO): @@ -696,6 +696,18 @@ def test_builtin_pager_short_output(toolbar_app, monkeypatch) -> None: assert "short output\n" in output.getvalue() +def test_builtin_pager_short_output_builds_no_pager(toolbar_app, monkeypatch) -> None: + app, _, output = toolbar_app + app.use_builtin_pager = True + monkeypatch.setattr(app, "stdin", Terminal()) + monkeypatch.setenv("TERM", "xterm") + # Output that fits is written directly, so none of the pager's widgets are needed. + with mock.patch("cmd2.command_toolbar.Pager") as pager, app._command_toolbar_context(): + app.ppaged("short output") + pager.assert_not_called() + assert "short output\n" in output.getvalue() + + def test_external_pager_suspends_shared_application(toolbar_app, monkeypatch) -> None: app, _, _ = toolbar_app app.use_builtin_pager = False @@ -736,9 +748,17 @@ def test_pager_styles_and_wrapping(chop) -> None: assert pager.text.read_only lexer = pager.text.lexer.lex_document(pager.text.document) assert "ansired" in lexer(0)[0][0] - assert not pager.fits(20, 1) - assert pager.fits(20, 5) is (not chop) - assert pager.fits(100, 1) + + +@pytest.mark.parametrize("chop", [False, True]) +def test_output_fits_measures_styled_and_wide_text(chop) -> None: + # Forty double-width characters occupy eighty columns, and styling them adds + # escape sequences that must not count towards the measurement. + text = "\x1b[31m" + "η•Œ" * 40 + "\x1b[0m\n" + assert not output_fits(text, 20, 1, chop=chop) + # Wrapping the line onto four rows fits; chopping keeps it one wide row that does not. + assert output_fits(text, 20, 5, chop=chop) is (not chop) + assert output_fits(text, 100, 1, chop=chop) @pytest.mark.parametrize("chop", [False, True]) From cc6ea8b51ed15177f7a5436f0fb5f91a48e79c03 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sat, 5 Sep 2026 23:34:09 -0400 Subject: [PATCH 08/35] Fixed 4 issues found during automated code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Chop-mode paging no longer loses your column β€” cmd2/pager.py:_scroll() The destination column now comes from the window's current horizontal_scroll when chop is set, instead of target * window_width (always 0 in chop mode) 2. A toolbar that can't start no longer takes the session down - cmd2/cmd2.py _command_toolbar_context() catches startup failure, reports once via perror, sets a new _command_toolbar_disabled flag so later commands don't retry and re-spam and runs the command without a toolbar 3. ppaged() no longer starts a toolbar outside the command loop β€” it now pages inside a toolbar only when one is already running, and otherwise falls through to the external pager. The documentation has been updated to contain truthful statements in this regard. 4. use_builtin_pager documented as opt-out only β€” initialization.md, os.md, prompt.md and the ppaged() docstring now state that it cannot turn the embedded pager on where no toolbar is running, since the two share one display. --- cmd2/cmd2.py | 49 ++++++--- cmd2/pager.py | 6 +- docs/features/initialization.md | 2 +- docs/features/os.md | 8 +- docs/features/prompt.md | 11 +- tests/test_command_toolbar.py | 181 ++++++++++++++++++++++++-------- 6 files changed, 188 insertions(+), 69 deletions(-) diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index 8d91aa2ed..3209628b2 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -560,6 +560,8 @@ def __init__( # to ensure they modify the correct session state. self.active_session = self.main_session self._command_toolbar: command_toolbar.CommandToolbar | None = None + # Set once a toolbar fails to start, so the failure is reported only once + self._command_toolbar_disabled = False # Commands to exclude from the history command self.exclude_from_history = ["_eof", "history"] @@ -1896,9 +1898,10 @@ def ppaged( fits on the screen. A pager is not used inside a script (Python or text) or when output is redirected or piped, and in these cases, output is sent to `poutput`. - With the bottom toolbar enabled, the built-in pager keeps it visible and refreshing. + While the bottom toolbar is running, the built-in pager keeps it visible and refreshing. Set ``use_builtin_pager=False`` to use the configured external ``pager`` or ``pager_chop`` - command instead; external pagers temporarily hide the toolbar. + command instead; external pagers temporarily hide the toolbar. Where no toolbar is + running, such as outside the command loop, the external pager is used regardless. :param chop: True -> causes lines longer than the screen width to be chopped (truncated) rather than wrapped - truncated text is still accessible by scrolling with the right & left arrow keys @@ -1950,11 +1953,12 @@ def ppaged( ) output = capture.get() - if self.use_builtin_pager: - with self._command_toolbar_context(): - if self._command_toolbar is not None and self._command_toolbar.is_active: - self._command_toolbar.page(output, chop=chop) - return + # Page inside the toolbar's display only when the command loop is already + # running one. Starting one here would seize the terminal for commands run + # outside that loop, which the toolbar is documented not to do. + if self.use_builtin_pager and self._command_toolbar is not None and self._command_toolbar.is_active: + self._command_toolbar.page(output, chop=chop) + return output_bytes = output.encode("utf-8", "replace") @@ -2097,17 +2101,28 @@ def _command_toolbar_context(self) -> Iterator[None]: """Display the toolbar around commands launched by the interactive command loop.""" if ( self._command_toolbar is not None + or self._command_toolbar_disabled or self.main_session.bottom_toolbar is None or not self._is_tty_session(self.main_session) ): yield return - toolbar = command_toolbar.CommandToolbar(self) try: with self.sigint_protection: + toolbar = command_toolbar.CommandToolbar(self) toolbar.start() self._command_toolbar = toolbar + except Exception as exc: # noqa: BLE001 + # The toolbar is cosmetic, so a display that cannot start must not take + # the command down, nor escape cmdloop() and leave its signal handlers + # installed. Report it once and run without it for the rest of the session. + self._command_toolbar_disabled = True + self.perror(f"Disabling the bottom toolbar during commands: {exc!r}") + yield + return + + try: yield finally: with self.sigint_protection: @@ -5988,20 +6003,22 @@ def cmdloop(self, intro: RenderableType = "") -> int: self.poutput(self.intro) # And then call _cmdloop() to enter the main loop - self._cmdloop() + try: + self._cmdloop() + finally: + # Restore original signal handlers however the loop ended. Leaving cmd2's + # handlers installed would outlive the application in its host process. + signal.signal(signal.SIGINT, original_sigint_handler) + + if not sys.platform.startswith("win"): + signal.signal(signal.SIGHUP, original_sighup_handler) + signal.signal(signal.SIGTERM, original_sigterm_handler) # Run the postloop() no matter what for func in self._postloop_hooks: func() self.postloop() - # Restore original signal handlers - signal.signal(signal.SIGINT, original_sigint_handler) - - if not sys.platform.startswith("win"): - signal.signal(signal.SIGHUP, original_sighup_handler) - signal.signal(signal.SIGTERM, original_sigterm_handler) - return self.exit_code ### diff --git a/cmd2/pager.py b/cmd2/pager.py index 97b9ec41e..1cf33f64c 100644 --- a/cmd2/pager.py +++ b/cmd2/pager.py @@ -203,7 +203,11 @@ def height(number: int) -> int: target -= height(line) line += 1 target = max(0, min(target, height(line) - 1)) - column = self._column_at_width(document.lines[line], target * info.window_width) + # Chopped lines never wrap, so leave the cursor at the column the reader + # scrolled to. Moving it to the start of the line would drag the view back + # with it, since the window scrolls horizontally to keep the cursor visible. + width = self.text.window.horizontal_scroll if self.chop else target * info.window_width + column = self._column_at_width(document.lines[line], width) event.current_buffer.cursor_position = document.translate_row_col_to_index(line, column) self.text.window.vertical_scroll = line self.text.window.vertical_scroll_2 = target if height(line) > info.window_height else 0 diff --git a/docs/features/initialization.md b/docs/features/initialization.md index 74b463174..88e6d79d7 100644 --- a/docs/features/initialization.md +++ b/docs/features/initialization.md @@ -52,7 +52,7 @@ Here are instance attributes of `cmd2.Cmd` which developers might wish to overri - **max_completion_table_items**: The maximum number of completion results allowed for a completion table to appear (Default: 50) - **pager**: sets the pager command used by the `Cmd.ppaged()` method for displaying wrapped output using a pager - **pager_chop**: sets the pager command used by the `Cmd.ppaged()` method for displaying chopped/truncated output using a pager -- **use_builtin_pager**: defaults to `enable_bottom_toolbar`; when enabled, `Cmd.ppaged()` uses an embedded pager with the persistent toolbar. Set to `False` to use the external `pager`/`pager_chop` commands. +- **use_builtin_pager**: defaults to `enable_bottom_toolbar`. While the command toolbar is running, `Cmd.ppaged()` uses an embedded pager that keeps that toolbar visible. Set to `False` to always use the external `pager`/`pager_chop` commands. This is an opt-*out* only: setting it to `True` does nothing unless `enable_bottom_toolbar` is also set, because the embedded pager shares the toolbar's display. - **py_bridge_name**: name by which embedded Python environments and scripts refer to the `cmd2` application by in order to call commands (Default: `app`) - **py_locals**: dictionary that defines specific variables/functions available in Python shells and scripts (provides more fine-grained control than making everything available with **self_in_py**) - **quiet**: if `True`, then completely suppress nonessential output (Default: `False`) diff --git a/docs/features/os.md b/docs/features/os.md index 22a495c81..36e9359d8 100644 --- a/docs/features/os.md +++ b/docs/features/os.md @@ -44,10 +44,12 @@ system. Output of any command can be displayed one page at a time using the [cmd2.Cmd.ppaged][] method. -When the bottom toolbar is enabled, `ppaged()` uses an embedded pager so the toolbar remains visible -and continues refreshing. It supports scrolling, search, and chopped lines. Set +While the bottom toolbar is running, `ppaged()` uses an embedded pager so the toolbar remains +visible and continues refreshing. It supports scrolling, search, and chopped lines. Set `self.use_builtin_pager = False` to use the configured external `pager`/`pager_chop` commands -instead. See [Bottom Toolbar](./prompt.md#bottom-toolbar) for controls and details. +instead. Anywhere the toolbar is not running, such as a command invoked outside the command loop, +`ppaged()` uses the external pager regardless of this setting. See +[Bottom Toolbar](./prompt.md#bottom-toolbar) for controls and details. Alternatively, a terminal pager can be invoked directly using the ability to run shell commands with the `!` shortcut like so: diff --git a/docs/features/prompt.md b/docs/features/prompt.md index a5fcd4d0a..7d95f906f 100644 --- a/docs/features/prompt.md +++ b/docs/features/prompt.md @@ -114,15 +114,16 @@ fast and use a lock when reading state that a command or another thread modifies ### Commands That Take Over the Terminal -With the toolbar enabled, `ppaged()` uses an embedded pager that keeps the same toolbar visible and -refreshing. Use Space or PageDown to advance, `b` or PageUp to go back, `/` to search, `n`/`N` to -repeat a search, and `q` to return to the command. Arrow keys navigate the output; `g`/`G` jump to -the beginning/end. Short output is printed directly above the toolbar. Chopped output supports +While the toolbar is running, `ppaged()` uses an embedded pager that keeps the same toolbar visible +and refreshing. Use Space or PageDown to advance, `b` or PageUp to go back, `/` to search, `n`/`N` +to repeat a search, and `q` to return to the command. Arrow keys navigate the output; `g`/`G` jump +to the beginning/end. Short output is printed directly above the toolbar. Chopped output supports horizontal scrolling, including on Windows. Set `self.use_builtin_pager = False` to use your configured external `pager`/`pager_chop` commands. The embedded pager provides basic navigation and search; applications that need additional `less` -features can use this opt-out. +features can use this opt-out. The setting only ever turns the embedded pager off: it cannot turn it +on where no toolbar is running, since the two share one display. cmd2 temporarily hides the toolbar for its input prompts, external pagers, Python environments, and shell commands. It also hides it while a command's output is piped to another process, since that diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index 7820deb64..3d900d776 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -18,7 +18,7 @@ from prompt_toolkit.output import DummyOutput from prompt_toolkit.shortcuts import PromptSession -from cmd2 import Cmd +from cmd2 import Cmd, command_toolbar from cmd2.pager import Pager, output_fits @@ -553,19 +553,59 @@ def test_command_toolbar_input_eof_is_not_reported(toolbar_app, capsys) -> None: assert app._command_toolbar is None -def test_command_toolbar_render_failure(toolbar_app) -> None: +def test_command_toolbar_startup_failure_still_runs_the_command(toolbar_app, capsys) -> None: app, _, output = toolbar_app def broken_toolbar(): raise ValueError("broken toolbar") app.main_session.bottom_toolbar = broken_toolbar - with pytest.raises(ValueError, match="broken toolbar"), app._command_toolbar_context(): - pytest.fail("Command should not run after a toolbar startup failure") # pragma: no cover + ran = [] + # The toolbar is cosmetic. A display that cannot start must not take the command + # with it, and must not escape cmdloop() and leave signal handlers installed. + with app._command_toolbar_context(): + ran.append(True) + app.poutput("command output") + + assert ran == [True] + assert "broken toolbar" in capsys.readouterr().err + assert "command output\n" in output.getvalue() assert app.stdout is output assert app._command_toolbar is None +def test_command_toolbar_is_not_retried_after_a_startup_failure(toolbar_app, capsys) -> None: + app, _, _ = toolbar_app + + def broken_toolbar(): + raise ValueError("broken toolbar") + + app.main_session.bottom_toolbar = broken_toolbar + with app._command_toolbar_context(): + pass + assert "broken toolbar" in capsys.readouterr().err + + # Without this, every later command repeats the same failure and the same message. + app.main_session.bottom_toolbar = "STATUS" + with mock.patch("cmd2.command_toolbar.CommandToolbar") as toolbar, app._command_toolbar_context(): + toolbar.assert_not_called() + assert capsys.readouterr().err == "" + + +def test_cmdloop_restores_signal_handlers_when_the_loop_fails(toolbar_app, monkeypatch) -> None: + import signal + + app, _, _ = toolbar_app + original = signal.getsignal(signal.SIGINT) + monkeypatch.setattr(app, "_cmdloop", mock.Mock(side_effect=RuntimeError("loop failed"))) + + with pytest.raises(RuntimeError, match="loop failed"): + app.cmdloop() + + # cmd2's handlers must not outlive the loop in the host process. + assert signal.getsignal(signal.SIGINT) is original + + @pytest.mark.parametrize("enabled", [False, True]) def test_command_toolbar_headless(enabled) -> None: app = Cmd(allow_cli_args=False, enable_bottom_toolbar=enabled) @@ -591,19 +631,21 @@ def command(line, **kwargs): @pytest.mark.parametrize( - ("layout", "error", "message"), + ("layout", "message"), [ - (Layout(Window()), TypeError, "Unsupported PromptSession layout"), - (Layout(HSplit([Window()])), RuntimeError, "Cannot locate PromptSession bottom toolbar"), + (Layout(Window()), "Unsupported PromptSession layout"), + (Layout(HSplit([Window()])), "Cannot locate PromptSession bottom toolbar"), ], ) -def test_command_toolbar_requires_the_prompt_toolbar(toolbar_app, layout, error, message) -> None: +def test_command_toolbar_requires_the_prompt_toolbar(toolbar_app, capsys, layout, message) -> None: app, _, output = toolbar_app # The display reuses the prompt's own toolbar container. If a future prompt-toolkit - # release moves it, say so instead of rendering something wrong. - with mock.patch.object(app.main_session, "layout", layout), pytest.raises(error, match=message): - app._command_toolbar_context().__enter__() + # release moves it, say so and keep running without a toolbar. + with mock.patch.object(app.main_session, "layout", layout), app._command_toolbar_context(): + app.poutput("command output") + assert message in capsys.readouterr().err assert app._command_toolbar is None + assert "command output\n" in output.getvalue() assert app.stdout is output @@ -708,6 +750,22 @@ def test_builtin_pager_short_output_builds_no_pager(toolbar_app, monkeypatch) -> assert "short output\n" in output.getvalue() +def test_builtin_pager_needs_an_already_running_toolbar(toolbar_app, monkeypatch) -> None: + app, _, _ = toolbar_app + app.use_builtin_pager = True + monkeypatch.setattr(app, "stdin", Terminal()) + monkeypatch.setenv("TERM", "xterm") + # Outside the command loop there is no toolbar to page inside. Starting one here + # would wrap the terminal streams and enter raw mode where the docs promise not to. + with ( + mock.patch.object(command_toolbar.CommandToolbar, "start") as start, + mock.patch("subprocess.Popen") as external, + ): + app.ppaged("row\n" * 200) + start.assert_not_called() + external.assert_called_once() + + def test_external_pager_suspends_shared_application(toolbar_app, monkeypatch) -> None: app, _, _ = toolbar_app app.use_builtin_pager = False @@ -816,45 +874,57 @@ def test_builtin_pager_does_not_capture_redirected_output(toolbar_app, monkeypat class PagerKeys: - """Send keys to the built-in pager and wait for each one to be handled.""" + """Send keys to the built-in pager and wait for the frame that reflects them.""" def __init__(self, app, pipe, pager) -> None: self.pipe = pipe self.pager = pager + self.presses = 0 self.states = [] self.updated = threading.Condition() - app.main_session.app.key_processor.after_key_press += self._record + ui = app.main_session.app + ui.key_processor.after_key_press += self._count + ui.after_render += self._record + + def _count(self, _) -> None: + with self.updated: + self.presses += 1 def _record(self, _) -> None: - # Read the pager's own buffer rather than the focused one, which is the - # search field while a search is being typed. + # Read the pager's own buffer, not the focused one, which is the search field + # while a search is being typed. Read it after rendering, because + # prompt-toolkit settles the window's scroll offsets while it draws. with self.updated: - self.states.append((self.pager.text.buffer.document.cursor_position_row, self.pager.text.window.horizontal_scroll)) + self.states.append( + ( + self.presses, + self.pager.text.buffer.document.cursor_position_row, + self.pager.text.window.horizontal_scroll, + ) + ) self.updated.notify_all() def press(self, keys, row, column=0) -> None: - """Send keys and wait until a resulting position matches, so steps stay ordered.""" + """Send keys and wait for a drawn frame that shows the expected position.""" with self.updated: - index = len(self.states) + handled = self.presses self.pipe.send_text(keys) deadline = time.monotonic() + 5 + index = 0 with self.updated: while True: while index < len(self.states): - state = self.states[index] + presses, *position = self.states[index] index += 1 - if state == (row, column): + if presses > handled and position == [row, column]: return remaining = deadline - time.monotonic() notified = remaining > 0 and self.updated.wait(remaining) - assert notified, f"pager ignored {keys!r}: wanted {(row, column)}, saw {self.states}" + assert notified, f"pager ignored {keys!r}: wanted {(row, column)}, saw {self.states[-3:]}" -@pytest.mark.parametrize("chop", [False, True]) -def test_pager_navigation_keys(toolbar_app, chop) -> None: - app, pipe, _ = toolbar_app - lines = [f"row {index:03d}" for index in range(100)] - lines[3] = "" # A line with no columns for a scroll target to be clamped against. +def drive_pager(app, pipe, text, *, chop, script) -> None: + """Page text and run script against its keys while the pager is displayed.""" created = [] entered = threading.Event() @@ -870,22 +940,7 @@ def observe(ui): def interact(): try: assert entered.wait(5) - keys = PagerKeys(app, pipe, created[0]) - keys.press("j", row=1) - keys.press("j", row=2) - keys.press("j", row=3) # Land on the empty line. - keys.press("k", row=2) # Moving up off it re-enters the line above. - keys.press("G", row=99) - keys.press("g", row=0) - keys.press("/row 05\n", row=50) - keys.press("n", row=51) - keys.press("N", row=50) - keys.press("?row 01\n", row=19) - keys.press("x", row=19) # Unbound keys are swallowed, not queued for the prompt. - # Horizontal scrolling applies only to chopped output, and stops at the - # end of a line shorter than the requested column. - keys.press("\x1b[C", row=19, column=len("row 019") if chop else 0) - keys.press("\x1b[D", row=19, column=0) + script(PagerKeys(app, pipe, created[0])) finally: pipe.send_text("q") @@ -893,11 +948,51 @@ def interact(): with mock.patch("cmd2.command_toolbar.Pager", side_effect=make_pager), ThreadPoolExecutor() as executor: interaction = executor.submit(interact) with app._command_toolbar_context(): - app._command_toolbar.page("\n".join(lines), chop=chop) + app._command_toolbar.page(text, chop=chop) interaction.result(timeout=10) assert get_typeahead(pipe) == [] +def test_pager_vertical_scrolling_keeps_horizontal_position(toolbar_app) -> None: + app, pipe, _ = toolbar_app + # Wide rows are what chopped output is for. Scrolling right to read a column and + # then moving down a row must not throw that column away. + text = "\n".join(f"row {index:03d} " + "col " * 40 for index in range(100)) + + def script(keys) -> None: + keys.press("\x1b[C", row=0, column=40) + keys.press("j", row=1, column=40) + keys.press("k", row=0, column=40) + + drive_pager(app, pipe, text, chop=True, script=script) + + +@pytest.mark.parametrize("chop", [False, True]) +def test_pager_navigation_keys(toolbar_app, chop) -> None: + app, pipe, _ = toolbar_app + lines = [f"row {index:03d}" for index in range(100)] + lines[3] = "" # A line with no columns for a scroll target to be clamped against. + + def script(keys) -> None: + keys.press("j", row=1) + keys.press("j", row=2) + keys.press("j", row=3) # Land on the empty line. + keys.press("k", row=2) # Moving up off it re-enters the line above. + keys.press("G", row=99) + keys.press("g", row=0) + keys.press("/row 05\n", row=50) + keys.press("n", row=51) + keys.press("N", row=50) + keys.press("?row 01\n", row=19) + keys.press("x", row=19) # Unbound keys are swallowed, not queued for the prompt. + # Horizontal scrolling applies only to chopped output, and stops at the + # end of a line shorter than the requested column. + keys.press("\x1b[C", row=19, column=len("row 019") if chop else 0) + keys.press("\x1b[D", row=19, column=0) + + drive_pager(app, pipe, "\n".join(lines), chop=chop, script=script) + + @pytest.mark.parametrize("chop", [False, True]) def test_pager_scrolling_before_first_render(chop) -> None: pager = Pager("row\n" * 100, chop=chop) From f759c1d0b1ead1a97ff3b1437b39a9638e11fd67 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sat, 5 Sep 2026 23:50:59 -0400 Subject: [PATCH 09/35] Fixed the pager to strip OSC sequences before ANSI parsing, preserving visible hyperlink text, colors, and width calculations When ppaged() receives Rich output containing hyperlinks, Rich emits OSC 8 sequences that prompt-toolkit's ANSI parser does not understand. Their contents become visible pager text: a linked click here renders as 8;id=...;https://example.comclick here8;;. This also corrupts width calculations and searchable text. This change strips unsupported OSC sequences before passing captured output to cmd2's new internal pager. --- cmd2/pager.py | 6 ++++++ tests/test_command_toolbar.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/cmd2/pager.py b/cmd2/pager.py index 1cf33f64c..438dd4e48 100644 --- a/cmd2/pager.py +++ b/cmd2/pager.py @@ -1,5 +1,6 @@ """A pager view hosted by the main prompt-toolkit application.""" +import re import threading from collections.abc import Callable from functools import partial @@ -20,9 +21,14 @@ from prompt_toolkit.utils import get_cwidth from prompt_toolkit.widgets import SearchToolbar, TextArea +_OSC_RE = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)") + def _fragments(text: str) -> StyleAndTextTuples: """Parse captured output, dropping the trailing newline that ends its last line.""" + # prompt-toolkit's ANSI parser does not handle OSC sequences, including Rich + # hyperlinks. Remove their metadata while retaining visible text and SGR styles. + text = _OSC_RE.sub("", text) return to_formatted_text(ANSI(text.removesuffix("\n"))) diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index 3d900d776..0fd97dbd4 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -17,6 +17,7 @@ from prompt_toolkit.layout import HSplit, Layout, Window from prompt_toolkit.output import DummyOutput from prompt_toolkit.shortcuts import PromptSession +from rich.console import Console from cmd2 import Cmd, command_toolbar from cmd2.pager import Pager, output_fits @@ -808,6 +809,24 @@ def test_pager_styles_and_wrapping(chop) -> None: assert "ansired" in lexer(0)[0][0] +@pytest.mark.parametrize("chop", [False, True]) +@pytest.mark.parametrize("terminator", ["\x1b\\", "\x07"], ids=["ST", "BEL"]) +def test_pager_rich_hyperlinks(chop, terminator) -> None: + console = Console(force_terminal=True, color_system="standard", no_color=False) + with console.capture() as capture: + console.print("[link=https://example.com][red]click here[/red][/link] after") + captured = capture.get() + assert "\x1b]8;" in captured + captured = captured.replace("\x1b\\", terminator) + + pager = Pager(captured, chop=chop) + assert pager.text.text == "click here after" + lexer = pager.text.lexer.lex_document(pager.text.document) + assert "ansired" in lexer(0)[0][0] + assert output_fits(captured, len("click here after"), 1, chop=chop) + assert not output_fits(captured, len("click here after") - 1, 1, chop=chop) + + @pytest.mark.parametrize("chop", [False, True]) def test_output_fits_measures_styled_and_wide_text(chop) -> None: # Forty double-width characters occupy eighty columns, and styling them adds From a0b2d9e6604b558d509d1741bb9412c416955b2f Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sat, 5 Sep 2026 23:58:35 -0400 Subject: [PATCH 10/35] Moved tests for pager.py to new test_pager.py file from test_command_toolbar.py Also: - Moved common code used by test_command_toolbar.py and test_pager.py to conftest.py Since the pager is effectively its own component, it feels cleaner to have the tests in their own file. --- tests/conftest.py | 50 +++++++ tests/test_command_toolbar.py | 260 +--------------------------------- tests/test_pager.py | 227 +++++++++++++++++++++++++++++ 3 files changed, 279 insertions(+), 258 deletions(-) create mode 100644 tests/test_pager.py diff --git a/tests/conftest.py b/tests/conftest.py index 3a37e9856..746c9f2a1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ """Cmd2 unit/functional testing""" +import io import sys from collections.abc import Callable from contextlib import redirect_stderr @@ -12,6 +13,11 @@ ) import pytest +from prompt_toolkit.application import create_app_session +from prompt_toolkit.data_structures import Size +from prompt_toolkit.input import create_pipe_input +from prompt_toolkit.output import DummyOutput +from prompt_toolkit.shortcuts import PromptSession import cmd2 from cmd2 import rich_utils as ru @@ -192,3 +198,47 @@ def autoload_command_sets_app(): @pytest.fixture def manual_command_sets_app(): return WithCommandSets(auto_load_commands=False) + + +class Terminal(io.StringIO): + """An in-memory stream that claims to be a terminal.""" + + def isatty(self) -> bool: + return True + + +class RecordingOutput(DummyOutput): + """A prompt-toolkit output that records everything written to a Terminal.""" + + def __init__(self, stream: Terminal) -> None: + self.stdout = stream + self.size = Size(rows=24, columns=80) + + def get_size(self) -> Size: + return self.size + + def write(self, data: str) -> None: + self.stdout.write(data) + + def write_raw(self, data: str) -> None: + self.stdout.write(data) + + +@pytest.fixture +def toolbar_app(): + app = cmd2.Cmd(allow_cli_args=False) + output = Terminal() + app.stdout = output + with create_pipe_input() as pipe: + terminal = RecordingOutput(output) + # Bind the ambient app session to this terminal. Without it, prompt-toolkit + # builds a real one on demand for calls such as patch_stdout() in + # _read_raw_input(), which needs a console that Windows CI does not provide. + with create_app_session(input=pipe, output=terminal): + app.main_session = PromptSession( + input=pipe, + output=terminal, + bottom_toolbar="STATUS", + refresh_interval=0.01, + ) + yield app, pipe, output diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index 0fd97dbd4..fa3fd7b47 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -1,6 +1,5 @@ """Command toolbar lifecycle and terminal integration tests.""" -import io import sys import threading import time @@ -9,58 +8,16 @@ from unittest import mock import pytest -from prompt_toolkit.application import create_app_session, get_app -from prompt_toolkit.data_structures import Size +from prompt_toolkit.application import get_app from prompt_toolkit.input import create_pipe_input from prompt_toolkit.input.typeahead import get_typeahead from prompt_toolkit.keys import Keys from prompt_toolkit.layout import HSplit, Layout, Window -from prompt_toolkit.output import DummyOutput from prompt_toolkit.shortcuts import PromptSession -from rich.console import Console from cmd2 import Cmd, command_toolbar -from cmd2.pager import Pager, output_fits - -class Terminal(io.StringIO): - def isatty(self) -> bool: - return True - - -class RecordingOutput(DummyOutput): - def __init__(self, stream: Terminal) -> None: - self.stdout = stream - self.size = Size(rows=24, columns=80) - - def get_size(self) -> Size: - return self.size - - def write(self, data: str) -> None: - self.stdout.write(data) - - def write_raw(self, data: str) -> None: - self.stdout.write(data) - - -@pytest.fixture -def toolbar_app(): - app = Cmd(allow_cli_args=False) - output = Terminal() - app.stdout = output - with create_pipe_input() as pipe: - terminal = RecordingOutput(output) - # Bind the ambient app session to this terminal. Without it, prompt-toolkit - # builds a real one on demand for calls such as patch_stdout() in - # _read_raw_input(), which needs a console that Windows CI does not provide. - with create_app_session(input=pipe, output=terminal): - app.main_session = PromptSession( - input=pipe, - output=terminal, - bottom_toolbar="STATUS", - refresh_interval=0.01, - ) - yield app, pipe, output +from .conftest import RecordingOutput, Terminal def test_command_toolbar_refresh_and_output(toolbar_app, monkeypatch) -> None: @@ -800,85 +757,6 @@ def close_input(ui): assert app.stdout is output -@pytest.mark.parametrize("chop", [False, True]) -def test_pager_styles_and_wrapping(chop) -> None: - pager = Pager("\x1b[31m" + "η•Œ" * 40 + "\x1b[0m\n", chop=chop) - assert pager.text.text == "η•Œ" * 40 - assert pager.text.read_only - lexer = pager.text.lexer.lex_document(pager.text.document) - assert "ansired" in lexer(0)[0][0] - - -@pytest.mark.parametrize("chop", [False, True]) -@pytest.mark.parametrize("terminator", ["\x1b\\", "\x07"], ids=["ST", "BEL"]) -def test_pager_rich_hyperlinks(chop, terminator) -> None: - console = Console(force_terminal=True, color_system="standard", no_color=False) - with console.capture() as capture: - console.print("[link=https://example.com][red]click here[/red][/link] after") - captured = capture.get() - assert "\x1b]8;" in captured - captured = captured.replace("\x1b\\", terminator) - - pager = Pager(captured, chop=chop) - assert pager.text.text == "click here after" - lexer = pager.text.lexer.lex_document(pager.text.document) - assert "ansired" in lexer(0)[0][0] - assert output_fits(captured, len("click here after"), 1, chop=chop) - assert not output_fits(captured, len("click here after") - 1, 1, chop=chop) - - -@pytest.mark.parametrize("chop", [False, True]) -def test_output_fits_measures_styled_and_wide_text(chop) -> None: - # Forty double-width characters occupy eighty columns, and styling them adds - # escape sequences that must not count towards the measurement. - text = "\x1b[31m" + "η•Œ" * 40 + "\x1b[0m\n" - assert not output_fits(text, 20, 1, chop=chop) - # Wrapping the line onto four rows fits; chopping keeps it one wide row that does not. - assert output_fits(text, 20, 5, chop=chop) is (not chop) - assert output_fits(text, 100, 1, chop=chop) - - -@pytest.mark.parametrize("chop", [False, True]) -def test_pager_long_line_navigation_resize_and_typeahead(toolbar_app, chop) -> None: - app, pipe, _ = toolbar_app - app.main_session.bottom_toolbar = "STATUS ONE\nSTATUS TWO" - entered, scrolled, resized = (threading.Event() for _ in range(3)) - - def observe(ui): - if not ui.full_screen: - return - # Check the rendered frame, not the stream of incremental terminal - # writes, to verify both toolbar rows survive navigation and resizing. - screen = ui.renderer._last_screen - size = ui.output.get_size() - bottom = "".join(screen.data_buffer[size.rows - 1][x].char for x in range(size.columns)) - assert bottom.startswith("STATUS TWO") - entered.set() - if ui.current_buffer.cursor_position > 0: - scrolled.set() - if size.columns == 60: - resized.set() - - def interact(): - try: - assert entered.wait(2) - pipe.send_text("\x1b[C" if chop else " ") - assert scrolled.wait(2) - app.main_session.output.size = Size(rows=20, columns=60) - app.main_session.app.invalidate() - assert resized.wait(2) - finally: - pipe.send_text("qnext\n") - - app.main_session.app.after_render += observe - with ThreadPoolExecutor() as executor: - interaction = executor.submit(interact) - with app._command_toolbar_context(): - app._command_toolbar.page("η•Œ" * 4000, chop=chop) - interaction.result(timeout=2) - assert app._read_raw_input("Next: ", app.main_session) == "next" - - def test_builtin_pager_does_not_capture_redirected_output(toolbar_app, monkeypatch, tmp_path) -> None: app, _, output = toolbar_app app.use_builtin_pager = True @@ -890,137 +768,3 @@ def test_builtin_pager_does_not_capture_redirected_output(toolbar_app, monkeypat pager.assert_not_called() assert "Cmd2 Commands" in target.read_text() assert "Cmd2 Commands" not in output.getvalue() - - -class PagerKeys: - """Send keys to the built-in pager and wait for the frame that reflects them.""" - - def __init__(self, app, pipe, pager) -> None: - self.pipe = pipe - self.pager = pager - self.presses = 0 - self.states = [] - self.updated = threading.Condition() - ui = app.main_session.app - ui.key_processor.after_key_press += self._count - ui.after_render += self._record - - def _count(self, _) -> None: - with self.updated: - self.presses += 1 - - def _record(self, _) -> None: - # Read the pager's own buffer, not the focused one, which is the search field - # while a search is being typed. Read it after rendering, because - # prompt-toolkit settles the window's scroll offsets while it draws. - with self.updated: - self.states.append( - ( - self.presses, - self.pager.text.buffer.document.cursor_position_row, - self.pager.text.window.horizontal_scroll, - ) - ) - self.updated.notify_all() - - def press(self, keys, row, column=0) -> None: - """Send keys and wait for a drawn frame that shows the expected position.""" - with self.updated: - handled = self.presses - self.pipe.send_text(keys) - deadline = time.monotonic() + 5 - index = 0 - with self.updated: - while True: - while index < len(self.states): - presses, *position = self.states[index] - index += 1 - if presses > handled and position == [row, column]: - return - remaining = deadline - time.monotonic() - notified = remaining > 0 and self.updated.wait(remaining) - assert notified, f"pager ignored {keys!r}: wanted {(row, column)}, saw {self.states[-3:]}" - - -def drive_pager(app, pipe, text, *, chop, script) -> None: - """Page text and run script against its keys while the pager is displayed.""" - created = [] - entered = threading.Event() - - def make_pager(*args, **kwargs): - pager = Pager(*args, **kwargs) - created.append(pager) - return pager - - def observe(ui): - if created and ui.full_screen and ui.layout.current_buffer is created[0].text.buffer: - entered.set() - - def interact(): - try: - assert entered.wait(5) - script(PagerKeys(app, pipe, created[0])) - finally: - pipe.send_text("q") - - app.main_session.app.after_render += observe - with mock.patch("cmd2.command_toolbar.Pager", side_effect=make_pager), ThreadPoolExecutor() as executor: - interaction = executor.submit(interact) - with app._command_toolbar_context(): - app._command_toolbar.page(text, chop=chop) - interaction.result(timeout=10) - assert get_typeahead(pipe) == [] - - -def test_pager_vertical_scrolling_keeps_horizontal_position(toolbar_app) -> None: - app, pipe, _ = toolbar_app - # Wide rows are what chopped output is for. Scrolling right to read a column and - # then moving down a row must not throw that column away. - text = "\n".join(f"row {index:03d} " + "col " * 40 for index in range(100)) - - def script(keys) -> None: - keys.press("\x1b[C", row=0, column=40) - keys.press("j", row=1, column=40) - keys.press("k", row=0, column=40) - - drive_pager(app, pipe, text, chop=True, script=script) - - -@pytest.mark.parametrize("chop", [False, True]) -def test_pager_navigation_keys(toolbar_app, chop) -> None: - app, pipe, _ = toolbar_app - lines = [f"row {index:03d}" for index in range(100)] - lines[3] = "" # A line with no columns for a scroll target to be clamped against. - - def script(keys) -> None: - keys.press("j", row=1) - keys.press("j", row=2) - keys.press("j", row=3) # Land on the empty line. - keys.press("k", row=2) # Moving up off it re-enters the line above. - keys.press("G", row=99) - keys.press("g", row=0) - keys.press("/row 05\n", row=50) - keys.press("n", row=51) - keys.press("N", row=50) - keys.press("?row 01\n", row=19) - keys.press("x", row=19) # Unbound keys are swallowed, not queued for the prompt. - # Horizontal scrolling applies only to chopped output, and stops at the - # end of a line shorter than the requested column. - keys.press("\x1b[C", row=19, column=len("row 019") if chop else 0) - keys.press("\x1b[D", row=19, column=0) - - drive_pager(app, pipe, "\n".join(lines), chop=chop, script=script) - - -@pytest.mark.parametrize("chop", [False, True]) -def test_pager_scrolling_before_first_render(chop) -> None: - pager = Pager("row\n" * 100, chop=chop) - # Keys can arrive before the first frame is drawn, when the window still has no - # rendered geometry to scroll against. - assert pager.text.window.render_info is None - event = SimpleNamespace(current_buffer=pager.text.buffer) - pager._scroll_page(event, pages=1.0) - pager._scroll(event, 1) - pager._scroll_horizontal(event, 1) - assert pager.text.buffer.cursor_position == 0 - assert pager.text.window.horizontal_scroll == 0 diff --git a/tests/test_pager.py b/tests/test_pager.py new file mode 100644 index 000000000..9e2b37a9f --- /dev/null +++ b/tests/test_pager.py @@ -0,0 +1,227 @@ +"""Tests for the built-in pager view.""" + +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace +from unittest import mock + +import pytest +from prompt_toolkit.data_structures import Size +from prompt_toolkit.input.typeahead import get_typeahead +from rich.console import Console + +from cmd2.pager import Pager, output_fits + + +@pytest.mark.parametrize("chop", [False, True]) +def test_pager_styles_and_wrapping(chop) -> None: + pager = Pager("\x1b[31m" + "η•Œ" * 40 + "\x1b[0m\n", chop=chop) + assert pager.text.text == "η•Œ" * 40 + assert pager.text.read_only + lexer = pager.text.lexer.lex_document(pager.text.document) + assert "ansired" in lexer(0)[0][0] + + +@pytest.mark.parametrize("chop", [False, True]) +@pytest.mark.parametrize("terminator", ["\x1b\\", "\x07"], ids=["ST", "BEL"]) +def test_pager_rich_hyperlinks(chop, terminator) -> None: + console = Console(force_terminal=True, color_system="standard", no_color=False) + with console.capture() as capture: + console.print("[link=https://example.com][red]click here[/red][/link] after") + captured = capture.get() + assert "\x1b]8;" in captured + captured = captured.replace("\x1b\\", terminator) + + pager = Pager(captured, chop=chop) + assert pager.text.text == "click here after" + lexer = pager.text.lexer.lex_document(pager.text.document) + assert "ansired" in lexer(0)[0][0] + assert output_fits(captured, len("click here after"), 1, chop=chop) + assert not output_fits(captured, len("click here after") - 1, 1, chop=chop) + + +@pytest.mark.parametrize("chop", [False, True]) +def test_output_fits_measures_styled_and_wide_text(chop) -> None: + # Forty double-width characters occupy eighty columns, and styling them adds + # escape sequences that must not count towards the measurement. + text = "\x1b[31m" + "η•Œ" * 40 + "\x1b[0m\n" + assert not output_fits(text, 20, 1, chop=chop) + # Wrapping the line onto four rows fits; chopping keeps it one wide row that does not. + assert output_fits(text, 20, 5, chop=chop) is (not chop) + assert output_fits(text, 100, 1, chop=chop) + + +@pytest.mark.parametrize("chop", [False, True]) +def test_pager_long_line_navigation_resize_and_typeahead(toolbar_app, chop) -> None: + app, pipe, _ = toolbar_app + app.main_session.bottom_toolbar = "STATUS ONE\nSTATUS TWO" + entered, scrolled, resized = (threading.Event() for _ in range(3)) + + def observe(ui): + if not ui.full_screen: + return + # Check the rendered frame, not the stream of incremental terminal + # writes, to verify both toolbar rows survive navigation and resizing. + screen = ui.renderer._last_screen + size = ui.output.get_size() + bottom = "".join(screen.data_buffer[size.rows - 1][x].char for x in range(size.columns)) + assert bottom.startswith("STATUS TWO") + entered.set() + if ui.current_buffer.cursor_position > 0: + scrolled.set() + if size.columns == 60: + resized.set() + + def interact(): + try: + assert entered.wait(2) + pipe.send_text("\x1b[C" if chop else " ") + assert scrolled.wait(2) + app.main_session.output.size = Size(rows=20, columns=60) + app.main_session.app.invalidate() + assert resized.wait(2) + finally: + pipe.send_text("qnext\n") + + app.main_session.app.after_render += observe + with ThreadPoolExecutor() as executor: + interaction = executor.submit(interact) + with app._command_toolbar_context(): + app._command_toolbar.page("η•Œ" * 4000, chop=chop) + interaction.result(timeout=2) + assert app._read_raw_input("Next: ", app.main_session) == "next" + + +class PagerKeys: + """Send keys to the built-in pager and wait for the frame that reflects them.""" + + def __init__(self, app, pipe, pager) -> None: + self.pipe = pipe + self.pager = pager + self.presses = 0 + self.states = [] + self.updated = threading.Condition() + ui = app.main_session.app + ui.key_processor.after_key_press += self._count + ui.after_render += self._record + + def _count(self, _) -> None: + with self.updated: + self.presses += 1 + + def _record(self, _) -> None: + # Read the pager's own buffer, not the focused one, which is the search field + # while a search is being typed. Read it after rendering, because + # prompt-toolkit settles the window's scroll offsets while it draws. + with self.updated: + self.states.append( + ( + self.presses, + self.pager.text.buffer.document.cursor_position_row, + self.pager.text.window.horizontal_scroll, + ) + ) + self.updated.notify_all() + + def press(self, keys, row, column=0) -> None: + """Send keys and wait for a drawn frame that shows the expected position.""" + with self.updated: + handled = self.presses + self.pipe.send_text(keys) + deadline = time.monotonic() + 5 + index = 0 + with self.updated: + while True: + while index < len(self.states): + presses, *position = self.states[index] + index += 1 + if presses > handled and position == [row, column]: + return + remaining = deadline - time.monotonic() + notified = remaining > 0 and self.updated.wait(remaining) + assert notified, f"pager ignored {keys!r}: wanted {(row, column)}, saw {self.states[-3:]}" + + +def drive_pager(app, pipe, text, *, chop, script) -> None: + """Page text and run script against its keys while the pager is displayed.""" + created = [] + entered = threading.Event() + + def make_pager(*args, **kwargs): + pager = Pager(*args, **kwargs) + created.append(pager) + return pager + + def observe(ui): + if created and ui.full_screen and ui.layout.current_buffer is created[0].text.buffer: + entered.set() + + def interact(): + try: + assert entered.wait(5) + script(PagerKeys(app, pipe, created[0])) + finally: + pipe.send_text("q") + + app.main_session.app.after_render += observe + with mock.patch("cmd2.command_toolbar.Pager", side_effect=make_pager), ThreadPoolExecutor() as executor: + interaction = executor.submit(interact) + with app._command_toolbar_context(): + app._command_toolbar.page(text, chop=chop) + interaction.result(timeout=10) + assert get_typeahead(pipe) == [] + + +def test_pager_vertical_scrolling_keeps_horizontal_position(toolbar_app) -> None: + app, pipe, _ = toolbar_app + # Wide rows are what chopped output is for. Scrolling right to read a column and + # then moving down a row must not throw that column away. + text = "\n".join(f"row {index:03d} " + "col " * 40 for index in range(100)) + + def script(keys) -> None: + keys.press("\x1b[C", row=0, column=40) + keys.press("j", row=1, column=40) + keys.press("k", row=0, column=40) + + drive_pager(app, pipe, text, chop=True, script=script) + + +@pytest.mark.parametrize("chop", [False, True]) +def test_pager_navigation_keys(toolbar_app, chop) -> None: + app, pipe, _ = toolbar_app + lines = [f"row {index:03d}" for index in range(100)] + lines[3] = "" # A line with no columns for a scroll target to be clamped against. + + def script(keys) -> None: + keys.press("j", row=1) + keys.press("j", row=2) + keys.press("j", row=3) # Land on the empty line. + keys.press("k", row=2) # Moving up off it re-enters the line above. + keys.press("G", row=99) + keys.press("g", row=0) + keys.press("/row 05\n", row=50) + keys.press("n", row=51) + keys.press("N", row=50) + keys.press("?row 01\n", row=19) + keys.press("x", row=19) # Unbound keys are swallowed, not queued for the prompt. + # Horizontal scrolling applies only to chopped output, and stops at the + # end of a line shorter than the requested column. + keys.press("\x1b[C", row=19, column=len("row 019") if chop else 0) + keys.press("\x1b[D", row=19, column=0) + + drive_pager(app, pipe, "\n".join(lines), chop=chop, script=script) + + +@pytest.mark.parametrize("chop", [False, True]) +def test_pager_scrolling_before_first_render(chop) -> None: + pager = Pager("row\n" * 100, chop=chop) + # Keys can arrive before the first frame is drawn, when the window still has no + # rendered geometry to scroll against. + assert pager.text.window.render_info is None + event = SimpleNamespace(current_buffer=pager.text.buffer) + pager._scroll_page(event, pages=1.0) + pager._scroll(event, 1) + pager._scroll_horizontal(event, 1) + assert pager.text.buffer.cursor_position == 0 + assert pager.text.window.horizontal_scroll == 0 From 379a4733ea6f7ca43f57c2efe855fff03fe2991a Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sun, 6 Sep 2026 00:10:29 -0400 Subject: [PATCH 11/35] Fix tests that were failing on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cause: rich's Style.render() skips the OSC 8 hyperlink wrapper entirely when legacy_windows=True. Console(force_terminal=True) on Windows auto-detects a legacy console, so the fixture string never contained \x1b]8; β€” the failure was in the test's setup, not in the pager. Fix (tests/test_pager.py:31): pass legacy_windows=False to the Console, so the fixture emits real hyperlink sequences on every platform. --- tests/test_pager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_pager.py b/tests/test_pager.py index 9e2b37a9f..3e155dbf8 100644 --- a/tests/test_pager.py +++ b/tests/test_pager.py @@ -26,7 +26,9 @@ def test_pager_styles_and_wrapping(chop) -> None: @pytest.mark.parametrize("chop", [False, True]) @pytest.mark.parametrize("terminator", ["\x1b\\", "\x07"], ids=["ST", "BEL"]) def test_pager_rich_hyperlinks(chop, terminator) -> None: - console = Console(force_terminal=True, color_system="standard", no_color=False) + # legacy_windows=False keeps rich from suppressing OSC 8 hyperlinks when the + # tests run against a legacy Windows console. + console = Console(force_terminal=True, color_system="standard", no_color=False, legacy_windows=False) with console.capture() as capture: console.print("[link=https://example.com][red]click here[/red][/link] after") captured = capture.get() From a0a742c5b2b46cc71e9af21248c765c495a72f74 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sun, 6 Sep 2026 09:08:37 -0400 Subject: [PATCH 12/35] Fix Windows test hang from mocking the event loop's call_soon_threadsafe test_command_toolbar_ui_call_reports_display_failure patched call_soon_threadsafe on the toolbar's live event loop and swallowed every call, not just the one it meant to drop. The patch was still installed while asyncio.run() tore the loop down, and asyncio resolves the default executor's join future through call_soon_threadsafe, so that report was dropped too. On Linux and macOS prompt_toolkit registers stdin with loop.add_reader, no default executor is ever created, and shutdown_default_executor() returns immediately. On Windows prompt_toolkit parks its WaitForMultipleObjects watcher in loop.run_in_executor(), so the executor exists and the loop waits on a future nothing resolves: 300 seconds on Python 3.12 and later, and forever before that, where shutdown_default_executor() takes no timeout. Windows CI went from 34s to 345s on 3.14, and hung for six hours on 3.11. Drop only the request made on the command thread and pass everything else through. The display still dies with the ValueError the test asserts on. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Ej3Nyn6vGngxYqm9FA1Jw5 --- tests/test_command_toolbar.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index fa3fd7b47..963e27695 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -454,12 +454,19 @@ def test_command_toolbar_ui_call_reports_display_failure(toolbar_app, capsys) -> schedule = loop.call_soon_threadsafe failed = threading.Event() - def die(*_args, **_kwargs) -> None: + def die(*args, **kwargs): # The display dies instead of running the queued callback, so the future - # the command is waiting on never resolves. - if not failed.is_set(): + # the command is waiting on never resolves. Only drop that one request, + # made here on the command thread. asyncio uses call_soon_threadsafe from + # its own threads, and on Windows the default executor's join is reported + # through it while asyncio.run() shuts the loop down. Swallowing that + # report strands the toolbar thread for 300 seconds, or forever before + # Python 3.12, where shutdown_default_executor() has no timeout. + if not failed.is_set() and threading.current_thread() is threading.main_thread(): failed.set() schedule(lambda: toolbar.app.exit(exception=ValueError("broken display"))) + return None + return schedule(*args, **kwargs) with ( mock.patch.object(loop, "call_soon_threadsafe", side_effect=die), From 2d23c3586dd579e3b9508c51816269b9ba61a62c Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sun, 6 Sep 2026 09:19:46 -0400 Subject: [PATCH 13/35] Add CLAUDE.md file to give Claude Code guidance --- CLAUDE.md | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..5d2ab6fee --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,122 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this +repository. + +## Project + +`cmd2` is a framework for building interactive command-line applications (REPLs) in Python. It +extends the stdlib `cmd` module and is built on **prompt-toolkit** (interactive input, key bindings, +completion, toolbars) and **Rich** / **rich-argparse** (all output rendering, help formatting, +theming). Requires Python >= 3.11. Environment/package management is **uv**. + +## Commands + +All work happens inside the uv-managed venv β€” run Python via `uv run ...`, and `uvx ...` for one-off +tools. Never `pip install`; if a new dependency is needed, state why before adding it. + +```bash +make install # create the venv, install prek git hooks + prettier (one time) +make check # lock check + prek (ruff format/lint, prettier, typos) + ty + mypy +make test # pytest with coverage over tests/ +make docs-test # verify the docs build cleanly (zensical) +make docs # build + serve docs with live reload +make help # list all targets +``` + +`make check`, `make test`, and `make docs-test` must all pass before committing. Run `make check` +before creating or modifying any `.py` file as well. + +Narrower targets: `make format`, `make lint`, `make mypy`, `make ty`, `make typecheck`. + +Running a single test or file (coverage flags come from `pyproject.toml` `addopts`, so pass +`--no-cov` when you just want a fast run): + +```bash +uv run pytest tests/test_cmd2.py --no-cov +uv run pytest tests/test_cmd2.py::test_base_help -x --no-cov +uv run pytest -k "toolbar" --no-cov +``` + +On Windows-sensitive code, note `make test` runs pytest under `python -Xutf8`. + +## Architecture + +### Command dispatch + +`cmd2/cmd2.py` (the `Cmd` class) is the core and by far the largest module. The flow for one line of +input is: + +1. `cmdloop()` β†’ `_cmdloop()` drives a prompt-toolkit `PromptSession` (created in + `_create_prompt_session`), not readline. Completion, history, lexing, and key bindings are all + supplied by prompt-toolkit adapters in `cmd2/pt_utils.py` (`Cmd2Completer`, `Cmd2History`, + `Cmd2Lexer`). +2. `onecmd_plus_hooks()` parses the line into a `Statement` (`cmd2/parsing.py` β€” handles shortcuts, + aliases, macros, terminators, multiline commands, redirection and pipe tokens), then runs the + plugin hook chain. +3. `_redirect_output()` swaps `self.stdout` for a file or subprocess pipe when the statement has + redirection, and restores it in the `finally` path. +4. `onecmd()` looks up and calls the `do_*` method. + +Hooks (`cmd2/plugin.py` dataclasses, registered via `register_postparsing_hook`, +`register_precmd_hook`, `register_postcmd_hook`, `register_cmdfinalization_hook`, +`register_preloop_hook`, `register_postloop_hook`) are the supported extension points; the legacy +`precmd`/`postcmd` overrides still exist but are weaker. + +### Argument parsing and completion + +Three layers, all producing a `Cmd2ArgumentParser`: + +- `cmd2/argparse_utils.py` β€” `Cmd2ArgumentParser` plus monkey-patching that extends argparse with + range `nargs` tuples, per-argument completion metadata, and subcommand records. +- `cmd2/decorators.py` β€” `@with_argparser`, `@with_argument_list`, `@with_category`, + `@as_subcommand_to` attach parsers/metadata to `do_*` methods. +- `cmd2/annotated.py` β€” the newer, still-experimental Typer-style path: `@with_annotated` builds a + parser from a function's type hints, with `Argument`/`Option` metadata via `typing.Annotated`. + +Parsers are built lazily and cached by the `CommandParsers` class in `cmd2.py` (`_build_parser`), so +that `CommandSet`-supplied subcommands can be attached and detached at runtime. +`cmd2/argparse_completer.py` walks a parser to produce tab completions; completion results are +`Completions`/`CompletionItem` objects from `cmd2/completion.py`. + +### Output and theming + +Never print directly β€” go through `Cmd.print_to` / `poutput` / `perror` / `pwarning` / `pfeedback` / +`ppaged`, which route to Rich consoles defined in `cmd2/rich_utils.py` (`Cmd2GeneralConsole`, +`Cmd2RichArgparseConsole`, `Cmd2ExceptionConsole`). Styling flows one way: `cmd2/styles.py` +(style-name StrEnum + defaults) β†’ `cmd2/theme.py` (single global theme, updated in-place for Rich +and exposed to prompt-toolkit through a `DynamicStyle`) β†’ `cmd2/pt_utils.py` (`rich_to_pt_style` +converts Rich styles into prompt-toolkit ones). Adding a color or style means touching +`styles.py`/`colors.py`/`theme.py`, not hardcoding ANSI. + +`cmd2/command_toolbar.py` and `cmd2/pager.py` are the trickiest parts of the codebase: they keep a +prompt-toolkit application alive _during_ synchronous command execution (bottom toolbar refresh, +paging) while stdout is proxied and redirection may be active. Changes there need care around +threads, signals, and Windows behavior. + +### Modularity + +`cmd2/command_set.py` (`CommandSet`) lets commands live in separate classes that are registered and +unregistered at runtime via `Cmd.register_command_set`; `cmd2/py_bridge.py` exposes the app to +embedded Python/IPython shells and `run_pyscript` while keeping isolation. + +## Conventions + +- Ruff is authoritative for format and lint (`ruff.toml`, line length 127, double quotes). Do not + suppress lint errors in code you write; broad ignores already exist for `examples/` and `tests/`. +- Both `mypy --strict` and `ty` must pass on the `cmd2` package. Full type annotations are required + on all library code (excluded: `tests/`, `examples/`, `docs/`). +- Docstrings are enforced by pydocstyle rules; public API items are documented in `docs/api/*.md` + and rendered by mkdocstrings, so keep docstrings accurate when changing signatures. +- Anything not documented under `docs/api/` is not public API (`cmd2/constants.py` says so + explicitly). +- Add user-visible changes to `CHANGELOG.md` under the current in-progress version heading. +- `main` is the branch for the next PATCH release; MAJOR/MINOR work happens on a branch named for + the target version. Releases are tagged and published from `main`. +- Do not commit spec, plan, or markdown documents without asking first. Save plans to + `~/.superpowers/plans/` rather than the project directory. + +## Commit conventions + +Never add "Co-Authored-By" lines to commits. Do not include Claude attribution in commit messages, +PR descriptions, or any git metadata. From a0b52d2f7bd180391d9affa1d301fcd84f030cca Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sun, 6 Sep 2026 10:06:47 -0400 Subject: [PATCH 14/35] Added info on new embedded pager to CHANGELOG and docs --- CHANGELOG.md | 6 +++++ docs/features/os.md | 50 ++++++++++++++++++++++++++++++++++------- docs/features/prompt.md | 15 +++++-------- 3 files changed, 53 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 870acc218..f8c9c29e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ - Enhancements - `enable_bottom_toolbar=True` now keeps the toolbar visible and refreshing during command execution + - Added an embedded pager which `Cmd.ppaged()` uses while the bottom toolbar is running, so the + toolbar stays visible and refreshing instead of the terminal being handed to an external + pager. It supports vertical and horizontal scrolling, incremental search, and chopped lines on + every platform, including Windows where the default external pager (`more`) always wraps. + Output which already fits on the screen is printed directly rather than paged. Set + `self.use_builtin_pager = False` to keep using the external `pager`/`pager_chop` commands. ## 4.2.3 (September 2, 2026) diff --git a/docs/features/os.md b/docs/features/os.md index 36e9359d8..0706c7a69 100644 --- a/docs/features/os.md +++ b/docs/features/os.md @@ -42,14 +42,48 @@ system. ## Terminal pagers -Output of any command can be displayed one page at a time using the [cmd2.Cmd.ppaged][] method. - -While the bottom toolbar is running, `ppaged()` uses an embedded pager so the toolbar remains -visible and continues refreshing. It supports scrolling, search, and chopped lines. Set -`self.use_builtin_pager = False` to use the configured external `pager`/`pager_chop` commands -instead. Anywhere the toolbar is not running, such as a command invoked outside the command loop, -`ppaged()` uses the external pager regardless of this setting. See -[Bottom Toolbar](./prompt.md#bottom-toolbar) for controls and details. +Output of any command can be displayed one page at a time using the [cmd2.Cmd.ppaged][] method. A +pager is only used when the terminal is interactive. Inside a script, or when the command's output +is redirected or piped, `ppaged()` falls back to `poutput()`. + +### Embedded pager + +While the [bottom toolbar](./prompt.md#bottom-toolbar) is running, `ppaged()` displays its output in +an embedded pager rather than handing the terminal to an external program. The toolbar stays visible +and keeps refreshing beneath the paged output, and `cmd2` never gives up control of the terminal. + +Output which already fits in the space above the toolbar is printed directly, so there is no pager +to dismiss. + +| Keys | Action | +| ---------------------------------- | ------------------------------------------------- | +| `Space`, `PageDown`, `f`, `Ctrl-F` | Forward one page | +| `b`, `PageUp`, `Ctrl-B` | Back one page | +| `d` / `u`, `Ctrl-D` / `Ctrl-U` | Forward / back half a page | +| `j` / `k`, `Down` / `Up`, `Enter` | Down / up one line | +| `h` / `l`, `Left` / `Right` | Scroll left / right (chopped output only) | +| `g` / `G`, `Home` / `End` | Jump to the beginning / end | +| `/` / `?` | Search forward / backward | +| `n` / `N` | Jump to the next / previous match | +| `q`, `Escape`, `Ctrl-C` | Close the pager and return to the running command | +| `Ctrl-Z` | Suspend to the background, where supported | + +Passing `chop=True` truncates lines longer than the screen width instead of wrapping them, and the +hidden columns remain reachable with the left and right arrow keys. Unlike the default external +pager on Windows (`more`), the embedded pager supports chopping on every platform. + +Rich styles in the paged output are preserved: colors are rendered while scrolling and searching, +and the visible text of a hyperlink is shown without its escape sequence metadata. + +### External pagers + +Set `self.use_builtin_pager = False` to always use the external commands configured in `self.pager` +and `self.pager_chop`. This is the opt-out for applications which need `less` features the embedded +pager does not provide. External pagers temporarily hide the toolbar and restore it on exit. + +The setting only ever turns the embedded pager off. Because the embedded pager shares the toolbar's +display, it is unavailable anywhere a toolbar is not running, such as a command invoked outside the +command loop, and `ppaged()` uses the external pager there regardless of this setting. Alternatively, a terminal pager can be invoked directly using the ability to run shell commands with the `!` shortcut like so: diff --git a/docs/features/prompt.md b/docs/features/prompt.md index 7d95f906f..ceebd063e 100644 --- a/docs/features/prompt.md +++ b/docs/features/prompt.md @@ -114,16 +114,11 @@ fast and use a lock when reading state that a command or another thread modifies ### Commands That Take Over the Terminal -While the toolbar is running, `ppaged()` uses an embedded pager that keeps the same toolbar visible -and refreshing. Use Space or PageDown to advance, `b` or PageUp to go back, `/` to search, `n`/`N` -to repeat a search, and `q` to return to the command. Arrow keys navigate the output; `g`/`G` jump -to the beginning/end. Short output is printed directly above the toolbar. Chopped output supports -horizontal scrolling, including on Windows. - -Set `self.use_builtin_pager = False` to use your configured external `pager`/`pager_chop` commands. -The embedded pager provides basic navigation and search; applications that need additional `less` -features can use this opt-out. The setting only ever turns the embedded pager off: it cannot turn it -on where no toolbar is running, since the two share one display. +While the toolbar is running, `ppaged()` uses an embedded pager which keeps the same toolbar visible +and refreshing instead of handing the terminal to an external pager. Short output is printed +directly above the toolbar, and longer output becomes a scrollable, searchable view. Set +`self.use_builtin_pager = False` to opt out and use your configured external `pager`/`pager_chop` +commands. See [Embedded pager](./os.md#embedded-pager) for the key bindings and full details. cmd2 temporarily hides the toolbar for its input prompts, external pagers, Python environments, and shell commands. It also hides it while a command's output is piped to another process, since that From c9395ea283601d9e92601bb4348fdd46bf8d3a39 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sun, 6 Sep 2026 12:18:19 -0400 Subject: [PATCH 15/35] Test additional keyboard shortcuts in embedded pager tests --- tests/test_pager.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_pager.py b/tests/test_pager.py index 3e155dbf8..8807b3f88 100644 --- a/tests/test_pager.py +++ b/tests/test_pager.py @@ -196,6 +196,13 @@ def test_pager_navigation_keys(toolbar_app, chop) -> None: lines[3] = "" # A line with no columns for a scroll target to be clamped against. def script(keys) -> None: + keys.press("\x1b[B", row=1) # Down arrow. + keys.press("\x1b[A", row=0) # Up arrow. + page_rows = keys.pager.text.window.render_info.window_height - 1 + keys.press("\x1b[6~", row=page_rows) # Page Down. + keys.press("\x1b[5~", row=0) # Page Up. + keys.press("\r", row=1) + keys.press("\x1b[A", row=0) keys.press("j", row=1) keys.press("j", row=2) keys.press("j", row=3) # Land on the empty line. From c77e08ad845f2f12b2d4994912fdfc47782a1243 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sun, 6 Sep 2026 12:35:18 -0400 Subject: [PATCH 16/35] Added examples/pager_diag.py for diagnosing embedded pager issues on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It now has three commands, in the order your colleague should try them: 1. slow β€” sleeps 5 seconds. Does TOOLBAR IS HERE stay visible the whole time? This is the discriminating test: if the toolbar isn't up during a command, the real bug is that CommandToolbar never starts on Windows, and the pager is just collateral damage. 2. longout β€” pages 500 lines. Built-in pager ends in q: quit; the external one shows -- More --. 3. diag β€” dumps the guard state to real stderr (so it survives whatever the pager does to the screen). The single most useful thing to report is the slow result plus the diag block β€” that alone should help narrow down the underlying root cause. --- examples/pager_diag.py | 76 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 examples/pager_diag.py diff --git a/examples/pager_diag.py b/examples/pager_diag.py new file mode 100755 index 000000000..9951e340e --- /dev/null +++ b/examples/pager_diag.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +"""Diagnostic for the built-in pager on Windows. + +Background: ``ppaged()`` uses cmd2's built-in pager only while the bottom toolbar is +running during a command. Otherwise it falls through to the external pager, which on +Windows is ``more``. This script reports which of those guards is failing. + +Run it from a real terminal (git bash, Windows Terminal, cmd.exe) and answer these: + +1. ``slow`` - does the bottom toolbar stay visible for the 5 seconds it runs? +2. ``longout`` - does the pager show a help line ending in ``q: quit`` (built-in), + or ``-- More --`` (Windows' external ``more``)? +3. ``diag`` - prints the state below to stderr. Copy the whole report. +4. ``quit`` + +The three lines that decide which pager runs are ``use_builtin_pager``, +``_command_toolbar``, and ``toolbar.is_active``. +""" + +import sys +import time + +import cmd2 + + +class PagerDiag(cmd2.Cmd): + """Minimal app with the bottom toolbar enabled, so the built-in pager is eligible.""" + + def __init__(self) -> None: + super().__init__(enable_bottom_toolbar=True) + + def get_bottom_toolbar(self): + """Show something unmistakable, so its presence or absence is obvious.""" + return "TOOLBAR IS HERE" + + def do_slow(self, _) -> None: + """Sleep 5 seconds. The bottom toolbar should stay visible the whole time.""" + time.sleep(5) + self.poutput("done") + + def do_longout(self, _) -> None: + """Page 500 wide lines. This should open the built-in pager, not `more`.""" + self.ppaged("\n".join(f"row {i:03d} " + "col " * 30 for i in range(500))) + + def do_diag(self, _) -> None: + """Report why ppaged() chooses the built-in pager or the external one.""" + toolbar = self._command_toolbar + report = [ + f"platform = {sys.platform}", + f"stdin.isatty() = {self.stdin.isatty()}", + f"stdout.isatty() = {self.stdout.isatty()}", + f"_redirecting = {self._redirecting}", + f"in_pyscript/script = {self.in_pyscript()} / {self.in_script()}", + f"use_builtin_pager = {self.use_builtin_pager}", + f"_command_toolbar = {toolbar!r}", + f"_toolbar_disabled = {self._command_toolbar_disabled}", + f"main_session.input = {type(self.main_session.input).__name__}", + f"main_session.output = {type(self.main_session.output).__name__}", + f"bottom_toolbar set = {self.main_session.bottom_toolbar is not None}", + ] + if toolbar is not None: + report += [ + f"toolbar.is_active = {toolbar.is_active}", + f"toolbar._proxy = {toolbar._proxy!r}", + f"toolbar.app.is_running = {toolbar.app.is_running}", + f"toolbar._thread alive = {toolbar._thread is not None and toolbar._thread.is_alive()}", + f"toolbar._error = {toolbar._error!r}", + ] + # Write straight to the real terminal so the report survives whatever the + # toolbar or a pager does to the screen afterwards. + sys.__stderr__.write("\n".join(report) + "\n") + sys.__stderr__.flush() + + +if __name__ == "__main__": + sys.exit(PagerDiag().cmdloop()) From b1fb5ee5f956e4934b68c13f194bd648810583cd Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sun, 6 Sep 2026 12:49:43 -0400 Subject: [PATCH 17/35] Remove pager_diag.py example which was used temporarily for troubleshooting --- examples/pager_diag.py | 76 ------------------------------------------ 1 file changed, 76 deletions(-) delete mode 100755 examples/pager_diag.py diff --git a/examples/pager_diag.py b/examples/pager_diag.py deleted file mode 100755 index 9951e340e..000000000 --- a/examples/pager_diag.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python -"""Diagnostic for the built-in pager on Windows. - -Background: ``ppaged()`` uses cmd2's built-in pager only while the bottom toolbar is -running during a command. Otherwise it falls through to the external pager, which on -Windows is ``more``. This script reports which of those guards is failing. - -Run it from a real terminal (git bash, Windows Terminal, cmd.exe) and answer these: - -1. ``slow`` - does the bottom toolbar stay visible for the 5 seconds it runs? -2. ``longout`` - does the pager show a help line ending in ``q: quit`` (built-in), - or ``-- More --`` (Windows' external ``more``)? -3. ``diag`` - prints the state below to stderr. Copy the whole report. -4. ``quit`` - -The three lines that decide which pager runs are ``use_builtin_pager``, -``_command_toolbar``, and ``toolbar.is_active``. -""" - -import sys -import time - -import cmd2 - - -class PagerDiag(cmd2.Cmd): - """Minimal app with the bottom toolbar enabled, so the built-in pager is eligible.""" - - def __init__(self) -> None: - super().__init__(enable_bottom_toolbar=True) - - def get_bottom_toolbar(self): - """Show something unmistakable, so its presence or absence is obvious.""" - return "TOOLBAR IS HERE" - - def do_slow(self, _) -> None: - """Sleep 5 seconds. The bottom toolbar should stay visible the whole time.""" - time.sleep(5) - self.poutput("done") - - def do_longout(self, _) -> None: - """Page 500 wide lines. This should open the built-in pager, not `more`.""" - self.ppaged("\n".join(f"row {i:03d} " + "col " * 30 for i in range(500))) - - def do_diag(self, _) -> None: - """Report why ppaged() chooses the built-in pager or the external one.""" - toolbar = self._command_toolbar - report = [ - f"platform = {sys.platform}", - f"stdin.isatty() = {self.stdin.isatty()}", - f"stdout.isatty() = {self.stdout.isatty()}", - f"_redirecting = {self._redirecting}", - f"in_pyscript/script = {self.in_pyscript()} / {self.in_script()}", - f"use_builtin_pager = {self.use_builtin_pager}", - f"_command_toolbar = {toolbar!r}", - f"_toolbar_disabled = {self._command_toolbar_disabled}", - f"main_session.input = {type(self.main_session.input).__name__}", - f"main_session.output = {type(self.main_session.output).__name__}", - f"bottom_toolbar set = {self.main_session.bottom_toolbar is not None}", - ] - if toolbar is not None: - report += [ - f"toolbar.is_active = {toolbar.is_active}", - f"toolbar._proxy = {toolbar._proxy!r}", - f"toolbar.app.is_running = {toolbar.app.is_running}", - f"toolbar._thread alive = {toolbar._thread is not None and toolbar._thread.is_alive()}", - f"toolbar._error = {toolbar._error!r}", - ] - # Write straight to the real terminal so the report survives whatever the - # toolbar or a pager does to the screen afterwards. - sys.__stderr__.write("\n".join(report) + "\n") - sys.__stderr__.flush() - - -if __name__ == "__main__": - sys.exit(PagerDiag().cmdloop()) From 7663cd712dcc4ef050fd9113214746c7d4f12ac9 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sun, 6 Sep 2026 12:58:42 -0400 Subject: [PATCH 18/35] Expanded pager tests to ensure the keyboard shortcuts behave as expected --- tests/test_pager.py | 85 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/tests/test_pager.py b/tests/test_pager.py index 8807b3f88..a2a4283ba 100644 --- a/tests/test_pager.py +++ b/tests/test_pager.py @@ -234,3 +234,88 @@ def test_pager_scrolling_before_first_render(chop) -> None: pager._scroll_horizontal(event, 1) assert pager.text.buffer.cursor_position == 0 assert pager.text.window.horizontal_scroll == 0 + + +@pytest.mark.parametrize("chop", [False, True]) +def test_pager_half_page_scrolling(toolbar_app, chop) -> None: + """Half-page keys move half of what the full-page keys move, and never zero rows.""" + app, pipe, _ = toolbar_app + lines = [f"row {index:03d}" for index in range(200)] + + def script(keys) -> None: + page_rows = max(1, keys.pager.text.window.render_info.window_height - 1) + half_rows = max(1, int(page_rows * 0.5)) + # A half page has to be a distinct, smaller step for this test to mean + # anything. The pager's own arithmetic is what decides the row it lands on. + assert 0 < half_rows < page_rows + keys.press("d", row=half_rows) + keys.press("d", row=2 * half_rows) + keys.press("u", row=half_rows) + keys.press("\x04", row=2 * half_rows) # Ctrl-D. + keys.press("\x15", row=half_rows) # Ctrl-U. + # Half-page steps stay half a page next to a full one taken from the same row. + keys.press("g", row=0) + keys.press("\x1b[6~", row=page_rows) # Page Down. + + drive_pager(app, pipe, "\n".join(lines), chop=chop, script=script) + + +@pytest.mark.parametrize("key", ["\x1b", "\x03"], ids=["escape", "ctrl-c"]) +def test_pager_search_abort_keys(toolbar_app, key) -> None: + """Escape and Ctrl-C abort a search rather than closing the pager. + + While the search field has focus the pager's own close bindings are filtered out, + so these keys have to reach prompt-toolkit's search bindings instead. Those are + registered explicitly so that they still work when the main prompt uses Vi mode. + """ + app, pipe, _ = toolbar_app + lines = [f"row {index:03d}" for index in range(100)] + + def script(keys) -> None: + keys.press("j", row=1) + # Typing a search previews it without moving the pager's own cursor. + keys.press("/row 05", row=1) + # Aborting restores the row the search started from instead of quitting. + keys.press(key, row=1) + # Focus is back on the pager, so ordinary navigation works again. + keys.press("j", row=2) + + drive_pager(app, pipe, "\n".join(lines), chop=False, script=script) + + +@pytest.mark.parametrize("key", ["\x1b", "\x03", "q"], ids=["escape", "ctrl-c", "q"]) +def test_pager_close_keys(toolbar_app, key) -> None: + """Each close key ends the pager without leaving the keystroke for the next prompt. + + Escape is bound eagerly, and it is also the first byte of every arrow and page + key. Closing on a bare Escape must therefore not come at the cost of the escape + sequences that arrive with more bytes behind them. + """ + app, pipe, _ = toolbar_app + entered = threading.Event() + closed = threading.Event() + + def observe(ui) -> None: + if ui.full_screen: + entered.set() + + def interact() -> None: + assert entered.wait(5), "pager never opened" + # Scroll first, so the pager is known to be reading keys before the close key. + pipe.send_text("j") + pipe.send_text(key) + if not closed.wait(5): + # Rescue the blocked main thread so this fails as an assertion, not a hang. + pipe.send_text("q") + raise AssertionError(f"{key!r} did not close the pager") + + app.main_session.app.after_render += observe + with ThreadPoolExecutor() as executor: + interaction = executor.submit(interact) + try: + with app._command_toolbar_context(): + app._command_toolbar.page("\n".join(f"row {index:03d}" for index in range(200)), chop=False) + finally: + closed.set() + interaction.result(timeout=10) + assert get_typeahead(pipe) == [] From dee4c9bbc5bcfcdefecc0c3f764d22cb9ce7d934 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Sun, 6 Sep 2026 18:26:08 -0400 Subject: [PATCH 19/35] feat: keep the bottom toolbar visible during nested prompts (#1750) read_input() and read_secret() each build their own PromptSession and passed no bottom_toolbar, while _read_raw_input() suspends the main display. The toolbar therefore vanished entirely for as long as a nested prompt was up, which is a visible inconsistency for a bar documented as persistent. Pass the toolbar to both sessions when the application has one, gated on main_session.bottom_toolbar being set so an app without a toolbar does not sprout one. Also inherit refresh_interval, so a clock or status display keeps ticking while the nested prompt waits rather than freezing at its first render. select() is deliberately left alone: prompt-toolkit's choice() accepts a toolbar but has no refresh_interval, so its toolbar would go stale while the selection sits idle. --- CHANGELOG.md | 4 +++ cmd2/cmd2.py | 4 +++ docs/features/prompt.md | 13 +++++--- tests/test_cmd2.py | 66 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4dd43922..64fcb2d35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ - Enhancements - `enable_bottom_toolbar=True` now keeps the toolbar visible and refreshing during command execution + - `Cmd.read_input()` and `Cmd.read_secret()` now keep the bottom toolbar visible while they wait + for input, refreshing at the same `refresh_interval` as the main prompt, instead of the + toolbar disappearing for the duration of the nested prompt. `Cmd.select()` is unchanged, since + prompt-toolkit's `choice()` offers no way to configure a refresh interval - Added an embedded pager which `Cmd.ppaged()` uses while the bottom toolbar is running, so the toolbar stays visible and refreshing instead of the terminal being handed to an external pager. It supports vertical and horizontal scrolling, incremental search, and chopped lines on diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index a78f98e03..6bbb5bd1f 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -3776,6 +3776,7 @@ def read_input( temp_session: PromptSession[str] = PromptSession( auto_suggest=self.main_session.auto_suggest, + bottom_toolbar=self.get_bottom_toolbar if self.main_session.bottom_toolbar is not None else None, color_depth=self.main_session.color_depth, complete_style=self.main_session.complete_style, complete_in_thread=self.main_session.complete_in_thread, @@ -3786,6 +3787,7 @@ def read_input( key_bindings=self.main_session.key_bindings, input=self.main_session.input, output=self.main_session.output, + refresh_interval=self.main_session.refresh_interval, style=self.main_session.style, ) @@ -3803,10 +3805,12 @@ def read_secret( :raises Exception: any other exceptions raised by prompt() """ temp_session: PromptSession[str] = PromptSession( + bottom_toolbar=self.get_bottom_toolbar if self.main_session.bottom_toolbar is not None else None, color_depth=self.main_session.color_depth, enable_suspend=self.main_session.enable_suspend, input=self.main_session.input, output=self.main_session.output, + refresh_interval=self.main_session.refresh_interval, style=self.main_session.style, ) diff --git a/docs/features/prompt.md b/docs/features/prompt.md index ceebd063e..2368ab180 100644 --- a/docs/features/prompt.md +++ b/docs/features/prompt.md @@ -120,10 +120,15 @@ directly above the toolbar, and longer output becomes a scrollable, searchable v `self.use_builtin_pager = False` to opt out and use your configured external `pager`/`pager_chop` commands. See [Embedded pager](./os.md#embedded-pager) for the key bindings and full details. -cmd2 temporarily hides the toolbar for its input prompts, external pagers, Python environments, and -shell commands. It also hides it while a command's output is piped to another process, since that -process may be interactive, as `less` and `fzf` are. It restores the toolbar when those operations -finish. +cmd2 temporarily hides the toolbar for external pagers, Python environments, and shell commands. It +also hides it while a command's output is piped to another process, since that process may be +interactive, as `less` and `fzf` are. It restores the toolbar when those operations finish. + +[cmd2.Cmd.read_input][] and [cmd2.Cmd.read_secret][] keep the toolbar visible, and it continues to +refresh at the same `refresh_interval` as the main prompt, so a clock or status display does not go +stale while your command waits for input. [cmd2.Cmd.select][] does not yet show one, because +prompt-toolkit's `choice()` has no way to configure a refresh interval and its toolbar would go +stale while the selection sits idle. For custom terminal UIs, calls to `input()`, or subprocesses your own command code starts, use [cmd2.Cmd.suspend_bottom_toolbar][]: diff --git a/tests/test_cmd2.py b/tests/test_cmd2.py index 6f4ae3656..1aa0dca81 100644 --- a/tests/test_cmd2.py +++ b/tests/test_cmd2.py @@ -2391,6 +2391,72 @@ def test_read_secret_eof(base_app, monkeypatch): base_app.read_secret("Secret: ") +def _capture_nested_session(app, call): + """Run `call` and return the real PromptSession cmd2 built for the nested prompt.""" + captured = [] + + def fake_read_raw_input(prompt, session, **kwargs): + captured.append(session) + return "typed" + + with mock.patch.object(app, "_read_raw_input", side_effect=fake_read_raw_input): + call() + assert len(captured) == 1 + return captured[0] + + +def test_read_input_session_shows_the_bottom_toolbar(base_app) -> None: + """A nested prompt keeps the persistent toolbar instead of dropping it. + + read_input() builds its own PromptSession, so without being told about the + toolbar the bar simply vanishes for as long as the nested prompt is up. + """ + base_app.main_session.bottom_toolbar = base_app.get_bottom_toolbar + + session = _capture_nested_session(base_app, lambda: base_app.read_input("Prompt> ")) + + assert session.bottom_toolbar == base_app.get_bottom_toolbar + + +def test_read_input_session_inherits_refresh_interval(base_app) -> None: + """A toolbar clock keeps ticking while the nested prompt waits for input.""" + base_app.main_session.bottom_toolbar = base_app.get_bottom_toolbar + base_app.main_session.refresh_interval = 0.25 + + session = _capture_nested_session(base_app, lambda: base_app.read_input("Prompt> ")) + + assert session.refresh_interval == 0.25 + + +def test_read_secret_session_shows_the_bottom_toolbar(base_app) -> None: + """Reading a secret keeps the persistent toolbar; the bar is not the secret.""" + base_app.main_session.bottom_toolbar = base_app.get_bottom_toolbar + + session = _capture_nested_session(base_app, lambda: base_app.read_secret("Secret: ")) + + assert session.bottom_toolbar == base_app.get_bottom_toolbar + + +def test_read_secret_session_inherits_refresh_interval(base_app) -> None: + """A toolbar clock keeps ticking while a secret is being entered.""" + base_app.main_session.bottom_toolbar = base_app.get_bottom_toolbar + base_app.main_session.refresh_interval = 0.25 + + session = _capture_nested_session(base_app, lambda: base_app.read_secret("Secret: ")) + + assert session.refresh_interval == 0.25 + + +@pytest.mark.parametrize("method", ["read_input", "read_secret"]) +def test_nested_prompt_has_no_toolbar_when_the_app_has_none(base_app, method) -> None: + """An app without a bottom toolbar must not sprout one at a nested prompt.""" + assert base_app.main_session.bottom_toolbar is None + + session = _capture_nested_session(base_app, lambda: getattr(base_app, method)("Prompt> ")) + + assert session.bottom_toolbar is None + + def test_read_input_passes_all_arguments_to_resolver(base_app): mock_choices = ["choice1", "choice2"] mock_provider = mock.MagicMock(name="provider") From eeef9fa59deef80e357f62427e79aed7fdc0e76e Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Mon, 7 Sep 2026 12:46:55 -0400 Subject: [PATCH 20/35] Make the test suite environment-independent, fix UTF-8 redirection, and a UI-call race (#1752) * Render test output independently of the caller's environment Rich and cmd2 consult several environment variables when deciding whether to emit styling, and the suite inherited them. Exporting any one of them made large numbers of unrelated tests fail depending on who ran the suite: NO_COLOR failed 15 tests, and FORCE_COLOR and TTY_COMPATIBLE 53 each. The failures look like product regressions, which makes them expensive to diagnose. Neutralize them for every test. Tests that exercise these variables set them explicitly, which still works because a test's own monkeypatching runs after the fixture. A guard test fails if any of them reaches a test again. * Write redirected and piped output as UTF-8 Command output is rendered by Rich and routinely contains non-ASCII, but redirection targets and pipes were opened with the locale's encoding. On any system whose default is not UTF-8 -- a Windows console using a legacy code page, for instance -- redirecting output raised UnicodeEncodeError, and the user was left with an empty file and advice to set PYTHONIOENCODING. Open both with UTF-8 explicitly. Two tests that read redirected output back were relying on the locale encoding for decoding as well, so they now name it too. * Fix a race that reported a timeout for a successful UI call _call_in_ui() polls the pending future with a 0.1s timeout and, on expiry, re-raises when the future is already done. That branch exists because concurrent.futures.TimeoutError is TimeoutError on Python 3.11+, so a callback raising a timeout of its own cannot be told apart by type from the poll expiring. Re-raising the caught exception conflates the two. When the callback completes in the window between the poll expiring and the future being inspected, the caller is told the call timed out even though it succeeded. Ask the future for its outcome instead: a callback that raised a timeout still propagates it, and one that produced a value now returns it. Found while investigating an intermittent failure of test_command_toolbar_ui_call_propagates_failures, which reproduced once in 60 runs before this change and not once in 120 after. The added regression test drives the interleaving deterministically rather than relying on timing. (cherry picked from commit 84bc19e8e66fc668cc91f612caa2afdde40122a5) * Run the pipe encoding test through this interpreter, not cat The test piped through `cat`, which cmd.exe does not provide. On a Windows system without Unix utilities installed it would fail before reaching the encoding behavior it exists to check -- and Windows is exactly what the UTF-8 redirection fix targets. Use a sys.executable pass-through instead, matching the pipe tests already in tests/test_command_toolbar.py. Reverting either the pipe or the redirect encoding still fails these tests. --- CHANGELOG.md | 4 +++ cmd2/cmd2.py | 19 ++++++++--- cmd2/command_toolbar.py | 7 +++- tests/conftest.py | 17 ++++++++++ tests/test_command_toolbar.py | 41 ++++++++++++++++++++--- tests/test_run_pyscript.py | 2 +- tests/test_suite_environment.py | 59 +++++++++++++++++++++++++++++++++ 7 files changed, 137 insertions(+), 12 deletions(-) create mode 100644 tests/test_suite_environment.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 64fcb2d35..c14a88a6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ ## 4.3.0 (TBD) - Bug Fixes + - Fixed output redirection and piping failing on systems whose default encoding is not UTF-8, + such as a Windows console using a legacy code page. Command output is rendered by Rich and + routinely contains non-ASCII, so redirecting it raised `UnicodeEncodeError` and left an empty + file behind. Redirection targets and pipes now use UTF-8 explicitly - Fixed the right prompt being redrawn beside every accepted command line in the scrollback. prompt-toolkit includes it in the final frame of each prompt, which is the frame left on the terminal; it is now hidden there, as the bottom toolbar already was, and stays on the live diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index 6bbb5bd1f..7d4b466c2 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -3403,9 +3403,11 @@ def _redirect_output(self, statement: Statement) -> utils.RedirectionSavedState: # Create a pipe with read and write sides read_fd, write_fd = os.pipe() - # Open each side of the pipe - subproc_stdin = open(read_fd) # noqa: SIM115 - new_stdout: TextIO = cast(TextIO, open(write_fd, "w")) # noqa: SIM115 + # Open each side of the pipe. Both ends are given an explicit encoding: + # command output is rendered by Rich and routinely contains non-ASCII, which + # the locale encoding cannot always represent. + subproc_stdin = open(read_fd, encoding="utf-8") # noqa: SIM115 + new_stdout: TextIO = cast(TextIO, open(write_fd, "w", encoding="utf-8")) # noqa: SIM115 # Create pipe process in a separate group to isolate our signals from it. If a Ctrl-C event occurs, # our sigint handler will forward it only to the most recent pipe process. This makes sure pipe @@ -3472,8 +3474,15 @@ def _redirect_output(self, statement: Statement) -> utils.RedirectionSavedState: # statement.output can only contain REDIRECTION_APPEND or REDIRECTION_OUTPUT mode = "a" if statement.redirector == constants.REDIRECTION_APPEND else "w" try: - # Use line buffering - new_stdout = cast(TextIO, open(su.strip_quotes(statement.redirect_to), mode=mode, buffering=1)) # noqa: SIM115 + # Use line buffering. The encoding is explicit rather than the + # locale's: command output is rendered by Rich and routinely contains + # non-ASCII, so on a non-UTF-8 system -- a default Windows console, + # for instance -- redirection would otherwise fail and leave an empty + # file behind. + new_stdout = cast( + TextIO, + open(su.strip_quotes(statement.redirect_to), mode=mode, buffering=1, encoding="utf-8"), # noqa: SIM115 + ) except OSError as ex: raise RedirectionError("Failed to redirect output") from ex diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index da8c1c881..96faacb31 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -376,7 +376,12 @@ def call() -> None: value = result.result(timeout=0.1) except FutureTimeoutError: if result.done(): - raise + # The callback finished while this poll was expiring, or raised a + # TimeoutError of its own -- indistinguishable here, because + # concurrent.futures.TimeoutError is TimeoutError on Python 3.11+. + # Ask the future for its outcome rather than re-raising this poll's + # timeout, which would report a failure for a call that succeeded. + return result.result() self._check_running() else: return value diff --git a/tests/conftest.py b/tests/conftest.py index 746c9f2a1..db1af0b63 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -86,6 +86,23 @@ def run_cmd(app: cmd2.Cmd, cmd: str) -> tuple[list[str], list[str]]: return normalize(out), normalize(err) +#: Environment variables that change how Rich and cmd2 render output. Left inherited, +#: they make unrelated tests fail depending on who runs the suite: NO_COLOR fails 15 +#: tests, FORCE_COLOR and TTY_COMPATIBLE 53 each. +COLOR_ENVIRONMENT = ("NO_COLOR", "FORCE_COLOR", "TTY_COMPATIBLE", "TTY_INTERACTIVE") + + +@pytest.fixture(autouse=True) +def neutral_color_environment(monkeypatch: pytest.MonkeyPatch) -> None: + """Render output the same way regardless of the caller's environment. + + Tests that exercise these variables set them explicitly, which still works because + a test's own monkeypatching runs after this fixture. + """ + for name in COLOR_ENVIRONMENT: + monkeypatch.delenv(name, raising=False) + + @pytest.fixture def base_app() -> cmd2.Cmd: return cmd2.Cmd(include_py=True, include_ipy=True) diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index 963e27695..b2cf2d099 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -3,7 +3,8 @@ import sys import threading import time -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future, ThreadPoolExecutor +from concurrent.futures import TimeoutError as FutureTimeoutError from types import SimpleNamespace from unittest import mock @@ -58,7 +59,7 @@ def test_command_toolbar_redirected_output(toolbar_app, tmp_path) -> None: destination = tmp_path / "help.txt" with app._command_toolbar_context(): app.onecmd_plus_hooks(f'help > "{destination}"') - text = destination.read_text() + text = destination.read_text(encoding="utf-8") assert "Cmd2 Commands" in text assert "STATUS" not in text assert "Cmd2 Commands" not in output.getvalue() @@ -79,7 +80,7 @@ def command(statement, **kwargs): app.onecmd_plus_hooks(f'custom > "{destination}"') app.poutput("terminal output") - assert destination.read_text() == "before\nduring\nafter\n" + assert destination.read_text(encoding="utf-8") == "before\nduring\nafter\n" assert "before" not in output.getvalue() assert "during" not in output.getvalue() assert "after" not in output.getvalue() @@ -134,7 +135,7 @@ def command(statement, **kwargs): assert running == [False] # A process given the terminal writes to it directly instead of through a captured pipe. assert readers[0]._proc.stdout is None - assert "PIPED" in destination.read_text() + assert "PIPED" in destination.read_text(encoding="utf-8") def test_command_toolbar_binary_output(toolbar_app) -> None: @@ -431,6 +432,36 @@ def fail(exception: BaseException) -> None: assert toolbar._call_in_ui(lambda: time.sleep(0.2) or "finished") == "finished" +def test_command_toolbar_ui_call_returns_a_result_that_lands_during_the_poll(toolbar_app, monkeypatch) -> None: + """A callback finishing while the poll expires must return its value, not a timeout. + + `concurrent.futures.TimeoutError` is `TimeoutError` on Python 3.11+, so the poll + expiring and the callback raising a timeout of its own are indistinguishable by type. + Re-raising the caught exception once the future is done therefore reports a timeout for + a call that actually succeeded. + """ + app, _, _ = toolbar_app + + class RacyFuture(Future): + """Completes, and only then reports the poll as having expired.""" + + def __init__(self) -> None: + super().__init__() + self._polled = False + + def result(self, timeout=None): # type: ignore[no-untyped-def] + if timeout is not None and not self._polled: + self._polled = True + super().result(timeout=5) # let the callback finish first + raise FutureTimeoutError # then act as though the poll had expired + return super().result(timeout) + + monkeypatch.setattr(command_toolbar, "Future", RacyFuture) + with app._command_toolbar_context(): + toolbar = app._command_toolbar + assert toolbar._call_in_ui(lambda: "finished") == "finished" + + def test_command_toolbar_ui_call_after_display_stopped(toolbar_app) -> None: app, pipe, _ = toolbar_app with app._command_toolbar_context(): @@ -773,5 +804,5 @@ def test_builtin_pager_does_not_capture_redirected_output(toolbar_app, monkeypat with mock.patch("cmd2.command_toolbar.Pager") as pager, app._command_toolbar_context(): app.onecmd_plus_hooks(f'help > "{target}"') pager.assert_not_called() - assert "Cmd2 Commands" in target.read_text() + assert "Cmd2 Commands" in target.read_text(encoding="utf-8") assert "Cmd2 Commands" not in output.getvalue() diff --git a/tests/test_run_pyscript.py b/tests/test_run_pyscript.py index 69b335fca..e4030ebde 100644 --- a/tests/test_run_pyscript.py +++ b/tests/test_run_pyscript.py @@ -246,7 +246,7 @@ def test_run_pyscript_print_redirection(base_app, request, tmp_path, capsys) -> out, err = capsys.readouterr() # Verify the output file contains what we expect from print() - content = pathlib.Path(out_file).read_text() + content = pathlib.Path(out_file).read_text(encoding="utf-8") # Look for everything written to self.stdout assert len(content.splitlines()) == 4 diff --git a/tests/test_suite_environment.py b/tests/test_suite_environment.py new file mode 100644 index 000000000..bb9cb8d1f --- /dev/null +++ b/tests/test_suite_environment.py @@ -0,0 +1,59 @@ +"""Guards that the suite renders output independently of the developer's environment. + +Rich and cmd2 both consult environment variables when deciding whether to emit styling. +Inheriting them makes large numbers of unrelated tests fail depending on who runs them, +which is expensive to diagnose because the failures look like product regressions. +""" + +import os +import sys + +import pytest + +import cmd2 + +#: Variables that change how output is rendered. Rich reads all of these; cmd2 reads +#: NO_COLOR directly. Tests that exercise them set them explicitly instead. +COLOR_ENVIRONMENT = ("NO_COLOR", "FORCE_COLOR", "TTY_COMPATIBLE", "TTY_INTERACTIVE") + + +@pytest.mark.parametrize("name", COLOR_ENVIRONMENT) +def test_color_environment_does_not_leak_into_tests(name: str) -> None: + """A developer exporting any of these must not change the suite's results.""" + assert name not in os.environ, ( + f"{name} leaked into the test environment; output-rendering assertions would depend on who is running the suite" + ) + + +class EncodingProbe(cmd2.Cmd): + """Reports the encoding of whatever stream output is currently going to.""" + + def do_show_encoding(self, _: str) -> None: + """Print the current output stream's encoding.""" + self.poutput(f"ENCODING={getattr(self.stdout, 'encoding', None)}") + + +def test_redirection_to_a_file_uses_utf8(tmp_path) -> None: + """cmd2 renders non-ASCII, so a redirect target must not use the locale encoding. + + Opened with the locale encoding, redirecting styled output raises UnicodeEncodeError + on any non-UTF-8 system -- which includes a default Windows console -- leaving the + user an empty file and an error. + """ + app = EncodingProbe(allow_cli_args=False) + target = tmp_path / "out.txt" + app.onecmd_plus_hooks(f'show_encoding > "{target}"') + assert "ENCODING=utf-8" in target.read_text(encoding="utf-8") + + +#: A pass-through filter, run with this interpreter so the test does not depend on Unix +#: utilities being installed. `cmd.exe` has no `cat`, and this fix exists for Windows. +PASS_THROUGH = "import sys; sys.stdin.reconfigure(encoding='utf-8'); sys.stdout.write(sys.stdin.read())" + + +def test_piping_uses_utf8(tmp_path) -> None: + """The same applies to the pipe the subprocess reads from.""" + app = EncodingProbe(allow_cli_args=False) + target = tmp_path / "piped.txt" + app.onecmd_plus_hooks(f'show_encoding | "{sys.executable}" -c "{PASS_THROUGH}" > "{target}"') + assert "ENCODING=utf-8" in target.read_text(encoding="utf-8") From 946083871173edc8e550bd1498fbdb1182bab6ca Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Mon, 7 Sep 2026 13:19:53 -0400 Subject: [PATCH 21/35] Describe the redirection bug's actual reach in the changelog Measured which Windows ANSI code pages can represent the box-drawing characters Rich emits: every cp125x code page fails, covering US and Western European, Central European, Cyrillic, Greek, Turkish, Hebrew, Arabic, Baltic and Vietnamese systems. Only the CJK double-byte code pages survive. Calling it a legacy code page was wrong: cp1252 is the default on a current, fully updated Windows 11, and Python only defaults to UTF-8 mode in 3.15. --- CHANGELOG.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c14a88a6d..48e92d434 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,11 @@ ## 4.3.0 (TBD) - Bug Fixes - - Fixed output redirection and piping failing on systems whose default encoding is not UTF-8, - such as a Windows console using a legacy code page. Command output is rendered by Rich and - routinely contains non-ASCII, so redirecting it raised `UnicodeEncodeError` and left an empty - file behind. Redirection targets and pipes now use UTF-8 explicitly + - Fixed output redirection and piping raising `UnicodeEncodeError` and leaving an empty file + behind. Command output is rendered by Rich and contains box-drawing characters, which no + `cp125x` Windows ANSI code page can represent, so redirecting or piping it failed. This + affects current Windows 11 with default settings on Python 3.11 through 3.14, not only older + systems. Redirection targets and pipes now use UTF-8 explicitly - Fixed the right prompt being redrawn beside every accepted command line in the scrollback. prompt-toolkit includes it in the final frame of each prompt, which is the frame left on the terminal; it is now hidden there, as the bottom toolbar already was, and stays on the live From 13c9ea708f724d250c79b2f4cff4faa84539c707 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:26:04 -0500 Subject: [PATCH 22/35] Write window titles on the command toolbar's event loop While a command runs, the bottom toolbar's display renders to the main session's output from its own thread. set_window_title() only found that display's loop through active_session, which still names the original session when main_session is replaced after construction, so a title set during a command could be written and flushed from the calling thread, unsynchronized with the toolbar's rendering. Treat the toolbar's display like a prompt: queue the title on its event loop while it runs, and write any title whose callback the loop didn't run once the display has stopped. --- cmd2/cmd2.py | 8 ++++ cmd2/command_toolbar.py | 3 ++ tests/test_command_toolbar.py | 70 ++++++++++++++++++++++++++++++++++- 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index 330ac2754..2f0176d96 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -6047,6 +6047,9 @@ def set_window_title(self, title: str) -> None: several titles are set before then, only the latest is written. Output errors are ignored so they can't disrupt the prompt. + The bottom toolbar's display counts as a prompt here. While it is shown during a command, the + title is written on its event loop in the same way. + :param title: the new window title """ with self._window_title_lock: @@ -6087,6 +6090,11 @@ def _get_prompt_loop(self) -> asyncio.AbstractEventLoop | None: if select_app is not None: apps.append(select_app) + # While a command runs, the bottom toolbar's display draws on this terminal from its own thread. + toolbar = self._command_toolbar + if toolbar is not None: + apps.append(toolbar.app) + for app in apps: loop = app.loop if loop is not None: diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index 1c2abb741..e441f1c95 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -327,6 +327,9 @@ def _pause(self) -> None: if self._thread is not None: self._thread.join() self._thread = None + # Write any window title whose callback the display's loop didn't run. The display + # has stopped, so this can't interleave with its rendering. + self.cmd._flush_pending_window_title_quietly() # Return the borrowed application to the main prompt, including on # proxy failures. The upstream toolbar owned a separate application. if self._display_stack is not None: diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index b2cf2d099..941e3eae3 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -9,11 +9,13 @@ from unittest import mock import pytest -from prompt_toolkit.application import get_app +from prompt_toolkit.application import create_app_session, get_app +from prompt_toolkit.data_structures import Size from prompt_toolkit.input import create_pipe_input from prompt_toolkit.input.typeahead import get_typeahead from prompt_toolkit.keys import Keys from prompt_toolkit.layout import HSplit, Layout, Window +from prompt_toolkit.output.vt100 import Vt100_Output from prompt_toolkit.shortcuts import PromptSession from cmd2 import Cmd, command_toolbar @@ -308,6 +310,72 @@ def slow_write(data: str) -> int: assert "in flight\n" in output.getvalue() +class TitleTerminal(Vt100_Output): + """A VT100 terminal which records the thread that wrote each window title.""" + + def __init__(self, stream: Terminal) -> None: + super().__init__(stream, lambda: Size(rows=24, columns=80), enable_cpr=False) + self.title_threads: dict[str, threading.Thread] = {} + self.title_written = threading.Event() + + def set_title(self, title: str) -> None: + self.title_threads[title] = threading.current_thread() + self.title_written.set() + super().set_title(title) + + +@pytest.fixture +def title_toolbar_app(): + # A real VT100 output, since window titles skip a DummyOutput in favor of the app's own stdout. + stream = Terminal() + with create_pipe_input() as pipe: + terminal = TitleTerminal(stream) + with create_app_session(input=pipe, output=terminal): + app = Cmd(allow_cli_args=False) + app.stdout = stream + app.main_session = PromptSession(input=pipe, output=terminal, bottom_toolbar="STATUS", refresh_interval=0.01) + yield app, terminal, stream + + +def test_command_toolbar_writes_window_titles_on_its_display_thread(title_toolbar_app) -> None: + app, terminal, stream = title_toolbar_app + with app._command_toolbar_context(): + display_thread = app._command_toolbar._thread + app.set_window_title("Working") + # Wait while the display is running, so its loop has the chance to write the title. + assert terminal.title_written.wait(2) + + # Written by the display, which renders the toolbar to the same terminal, rather than by the command. + assert terminal.title_threads["Working"] is display_thread + assert "\x1b]2;Working\x07" in stream.getvalue() + + +def test_command_toolbar_writes_a_queued_title_when_its_display_stops(title_toolbar_app, monkeypatch) -> None: + app, terminal, stream = title_toolbar_app + dropped = [] + with app._command_toolbar_context(): + loop = app._command_toolbar.app.loop + schedule = loop.call_soon_threadsafe + + # Simulate the display stopping before its loop runs the title callback. + def drop_title_callback(callback, *args, **kwargs): + if getattr(callback, "__func__", None) is Cmd._flush_pending_window_title_quietly: + dropped.append(callback) + return None + return schedule(callback, *args, **kwargs) + + monkeypatch.setattr(loop, "call_soon_threadsafe", drop_title_callback) + app.set_window_title("Queued") + assert "Queued" not in terminal.title_threads + + assert len(dropped) == 1 + # Stopping the display wrote the title once the display could no longer render. + assert terminal.title_threads["Queued"] is threading.main_thread() + assert stream.getvalue().endswith("\x1b]2;Queued\x07") + assert app._pending_window_title is None + assert app._window_title_callback_loop is None + + def test_command_toolbar_typeahead(toolbar_app) -> None: app, pipe, _ = toolbar_app received = threading.Event() From 114364f228eafcd9834efcd0a8ca5249391923f1 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:28:17 -0500 Subject: [PATCH 23/35] Suspend the toolbar while the editor runs run_editor() now calls _run_shell_command() directly instead of going through do_shell(), so the suspension on do_shell() no longer covered the editor. Suspend around _run_shell_command(), which both callers share. --- cmd2/cmd2.py | 2 +- tests/test_command_toolbar.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index ad5d67d37..397fbf5ad 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -5109,7 +5109,6 @@ def _build_shell_parser(cls) -> Cmd2ArgumentParser: # Preserve quotes since we are passing these strings to the shell @with_argparser(_build_shell_parser, preserve_quotes=True) - @command_toolbar.suspend_toolbar def do_shell(self, args: argparse.Namespace) -> None: """Execute a command as if at the OS prompt.""" # Create a list of arguments to shell @@ -5119,6 +5118,7 @@ def do_shell(self, args: argparse.Namespace) -> None: utils.expand_user_in_tokens(tokens) self._run_shell_command(" ".join(tokens)) + @command_toolbar.suspend_toolbar def _run_shell_command(self, command: str) -> None: """Run a command line in the OS shell and wait for it to finish. diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index 941e3eae3..a96848e19 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -250,6 +250,31 @@ def test_command_toolbar_suspension_and_nested_input(toolbar_app) -> None: assert toolbar.app.is_running +@pytest.mark.parametrize("launch", ["shell", "editor"]) +def test_command_toolbar_suspended_for_shell_and_editor(toolbar_app, launch) -> None: + """The shell command and the editor both hand the terminal to another program.""" + app, _, _ = toolbar_app + app.editor = "vim" + running = [] + + def popen(*args, **kwargs): + running.append(app._command_toolbar.app.is_running) + return mock.Mock(returncode=0) + + with ( + mock.patch("subprocess.Popen", side_effect=popen), + mock.patch("cmd2.utils.ProcReader"), + app._command_toolbar_context(), + ): + if launch == "shell": + app.do_shell("ls") + else: + app.run_editor("notes.txt") + assert app._command_toolbar.app.is_running + + assert running == [False] + + class CprOutput(RecordingOutput): """A terminal that asks for cursor position reports and never answers them.""" From 189ab53ceb85ab7f3ca9df04287599bb3b19ae1c Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:17:13 -0500 Subject: [PATCH 24/35] Hold the display thread until a Ctrl-Z stop has taken it Ctrl-Z in the built-in pager, or at the command toolbar, sends SIGTSTP from the display thread. A process-directed signal lands on the main thread, and on Linux the display thread carries on for a few milliseconds: it restores raw mode, reattaches its input and redraws before the job has stopped. When the shell resumes the job it puts back the terminal modes it saved beforehand, and nothing is left to undo that. The terminal stays cooked, keys are held until Enter, and with a refreshing toolbar every cursor-position query is echoed as ^[[row;colR. After sending the signal, wait for the stop to take the sending thread as well. A stop shows as time passing while asleep; a signal that was ignored, or discarded for an orphaned process group, shows as nothing and the wait ends on its own. This is the fix made in 66513f6c on stage4-handoffs-integration, without its reserved toolbar parts. The regression test drives the pager under an interactive bash on a pty, suspends and resumes it, and requires that it still answers q. It fails on Linux without this change. --- cmd2/command_toolbar.py | 39 +++++++++++- tests/test_command_toolbar.py | 45 ++++++++++++++ tests/test_pipeline_job_control.py | 97 ++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 1 deletion(-) diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index e441f1c95..d42630317 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -8,12 +8,13 @@ import signal import sys import threading +import time from collections.abc import Callable, Generator from concurrent.futures import Future from concurrent.futures import TimeoutError as FutureTimeoutError from typing import TYPE_CHECKING, Any, TextIO, TypeVar, cast -from prompt_toolkit.application import Application, create_app_session +from prompt_toolkit.application import Application, create_app_session, run_in_terminal from prompt_toolkit.enums import EditingMode from prompt_toolkit.filters import Condition, to_filter from prompt_toolkit.input.typeahead import get_typeahead, store_typeahead @@ -33,6 +34,33 @@ _R = TypeVar("_R") +def suspend_process_group(suspend_group: bool = True) -> None: + """Stop this process, or its whole process group, with SIGTSTP from any thread. + + A process-directed stop signal can be taken by a thread other than the sender. Sent + from the display thread, it lands on the main thread, and the display thread carries + on for a few milliseconds: it puts the terminal back into raw mode and redraws before + the job has stopped. When the shell resumes the job it restores the modes it saved + beforehand, and nothing is left to undo that -- the terminal stays cooked, keys are + held until Enter, and every cursor-position query is echoed as ``^[[row;colR``. + + So after sending the signal this thread waits for the stop to take it too. A stop + shows as time passing while asleep; a signal that was ignored, or discarded for an + orphaned process group, shows as nothing, and the wait ends on its own. + + :param suspend_group: stop the whole process group, as a shell's Ctrl-Z would, rather + than only this process + """ + if not hasattr(signal, "SIGTSTP"): # pragma: no cover - POSIX only + return + os.kill(0 if suspend_group else os.getpid(), signal.SIGTSTP) + deadline = time.monotonic() + 0.25 + while (before := time.monotonic()) < deadline: + time.sleep(0.01) + if time.monotonic() - before > 0.1: + return + + def suspend_toolbar(func: _F) -> _F: """Give a method exclusive access to the terminal.""" @@ -144,6 +172,10 @@ def __init__(self, cmd: "Cmd") -> None: session = cmd.main_session self.app = session.app + # Ctrl-Z during a command is handled on the display's thread, where upstream's + # version would not hold the thread until the job has stopped. + if "suspend_to_background" not in vars(self.app): + cast("Any", self.app).suspend_to_background = self._suspend_to_background # PromptSession has no public hook for replacing just its input area. # Keep this small dependency on its layout shape in one place, and fail # explicitly if upstream changes it. Reuse the actual toolbar container, @@ -199,6 +231,11 @@ def suspend(event: KeyPressEvent) -> None: self._bindings = bindings self._suspend_binding = suspend + @staticmethod + def _suspend_to_background(suspend_group: bool = True) -> None: + """Suspend like upstream's ``Application.suspend_to_background()``, from any thread.""" + run_in_terminal(functools.partial(suspend_process_group, suspend_group)) + def _after_render(self, app: Application[str]) -> None: # noqa: ARG002 self._ready.set() diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index a96848e19..43c6b9c18 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -212,6 +212,51 @@ def test_command_toolbar_ctrl_z(toolbar_app, supported, enabled) -> None: assert [key.key for key in keys] == ([] if supported and enabled else [Keys.ControlZ]) +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX job control") +@pytest.mark.parametrize("stopped", [True, False]) +def test_suspend_process_group_waits_for_the_stop_to_take_its_thread(stopped) -> None: + """The sender sleeps until the stop reaches it, or gives up once nothing has happened. + + A stop shows as time passing while asleep, so a clock that jumps across the first sleep + stands in for a job that was stopped and then resumed. A signal that was ignored, or + discarded for an orphaned group, lets the clock advance only by what was slept. + """ + import signal + + clock = [100.0] + sleeps = [] + + def sleep(seconds): + sleeps.append(seconds) + clock[0] += 5.0 if stopped and len(sleeps) == 1 else seconds + + with ( + mock.patch("cmd2.command_toolbar.time", SimpleNamespace(monotonic=lambda: clock[0], sleep=sleep)), + mock.patch("cmd2.command_toolbar.os.kill") as kill, + ): + command_toolbar.suspend_process_group() + kill.assert_called_once_with(0, signal.SIGTSTP) + if stopped: + assert sleeps == [0.01] + else: + assert len(sleeps) > 1 + assert clock[0] >= 100.25 - 1e-6 + + +def test_command_toolbar_suspends_through_its_own_stop(toolbar_app) -> None: + """Ctrl-Z during a command stops the process with the helper which holds the display's thread.""" + app, _, _ = toolbar_app + with app._command_toolbar_context(): + toolbar = app._command_toolbar + assert vars(toolbar.app)["suspend_to_background"] == toolbar._suspend_to_background + with mock.patch("cmd2.command_toolbar.run_in_terminal") as run_in_terminal: + toolbar.app.suspend_to_background(suspend_group=False) + + (stop,) = run_in_terminal.call_args.args + assert stop.func is command_toolbar.suspend_process_group + assert stop.args == (False,) + + def test_command_toolbar_script_output_has_no_batching_delay(toolbar_app) -> None: app, _, output = toolbar_app sleep = mock.Mock() diff --git a/tests/test_pipeline_job_control.py b/tests/test_pipeline_job_control.py index cf6dcf4bb..1ad671faf 100644 --- a/tests/test_pipeline_job_control.py +++ b/tests/test_pipeline_job_control.py @@ -507,6 +507,103 @@ def wait_until(predicate): process.wait(timeout=5) +def test_pager_resumes_with_raw_terminal_after_suspend(tmp_path) -> None: + """Ctrl-Z in the built-in pager, then fg, must bring the pager back, not echoed cursor reports. + + The display thread sends the stop signal, which the kernel may hand to the main + thread. Unless the sending thread stops synchronously too, it restores raw mode + before the job has stopped, the shell puts its own modes back on fg, and every + cursor-position query afterwards is echoed as ^[[row;colR. + """ + import fcntl + import pty + import struct + import termios + + shell = shutil.which("bash") + if shell is None: + pytest.skip("requires an interactive bash shell") + application = tmp_path / "application.py" + application.write_text( + "from cmd2 import Cmd\n" + "class App(Cmd):\n" + " def do_page(self, statement):\n" + " self.ppaged('\\n'.join(f'line {n:04d}' for n in range(400)))\n" + # A refreshing toolbar, as in the examples: its periodic redraws are what turn a cooked terminal into a stream of echoed + # cursor reports. + "app = App(enable_bottom_toolbar=True, refresh_interval=0.5)\n" + "app.prompt = 'TEST> '\n" + "app.main_session.bottom_toolbar = 'STATUS'\n" + "app.cmdloop()\n", + encoding="utf-8", + ) + master, slave = pty.openpty() + fcntl.ioctl(slave, termios.TIOCSWINSZ, struct.pack("HHHH", 24, 80, 0, 0)) + bootstrap = ( + "import os, fcntl, termios; os.setsid(); " + "fcntl.ioctl(0, termios.TIOCSCTTY, 0); " + "os.execv(os.environ['TEST_SHELL'], ['bash', '--noprofile', '--norc', '-i'])" + ) + env = dict(os.environ, TERM="xterm-256color", PS1="OUTER> ", TEST_SHELL=shell, SHELL=shell) + env["PYTHONPATH"] = str(Path(__file__).resolve().parents[1]) + process = subprocess.Popen([sys.executable, "-c", bootstrap], stdin=slave, stdout=slave, stderr=slave, env=env) + os.close(slave) + screen = pyte.Screen(80, 24) + screen.write_process_input = lambda data: os.write(master, data.encode()) + stream = pyte.Stream(screen) + decoder = codecs.getincrementaldecoder("utf-8")("replace") + transcript = "" + + def send(data): + os.write(master, data.encode()) + + def pump(seconds, predicate=lambda: False): + nonlocal transcript + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if select.select([master], [], [], 0.05)[0]: + data = decoder.decode(os.read(master, 65536)) + transcript += data + stream.feed(data) + if predicate(): + return True + return predicate() + + def pager_shown(): + return any("q: quit" in line for line in screen.display) + + try: + assert pump(10, lambda: "OUTER> " in transcript) + send(f"{shlex.quote(sys.executable)} {shlex.quote(str(application))}\n") + assert pump(10, lambda: "TEST>" in "\n".join(screen.display)), transcript + # Let the prompt finish its cursor-position handshake before typing at it. + pump(0.5) + send("page\n") + assert pump(10, pager_shown), transcript + start = len(transcript) + send("\x1a") + assert pump(10, lambda: "OUTER> " in transcript[start:]), transcript + start = len(transcript) + send("fg\n") + assert pump(10, pager_shown), transcript[start:] + # Give a broken resume time to start looping before judging the transcript. + pump(1.5) + resumed = transcript[start:] + echoed = re.findall(r"\^\[\[\d+;\d+R", resumed) + assert not echoed, f"cursor position reports were echoed after fg: {echoed[:5]}\n{resumed}" + assert pager_shown(), "\n".join(screen.display) + # A cooked terminal would hold the key back until Enter: the pager must quit on q alone. + start = len(transcript) + send("q") + assert pump(10, lambda: "TEST>" in transcript[start:]), f"pager did not quit on q after fg:\n{transcript[start:]}" + send("quit\n") + pump(5, lambda: os.tcgetpgrp(master) == process.pid) + finally: + os.close(master) + process.kill() + process.wait(timeout=5) + + @pytest.mark.parametrize("parent_delay", [False, True]) def test_pipeline_pager_can_set_terminal_modes_at_startup(tmp_path, parent_delay) -> None: """A pager such as less puts the terminal in raw mode as it starts, before reading its pipe. From 47308238939cafdcfb6259531031e2af31d37ebb Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:17:16 -0500 Subject: [PATCH 25/35] Stop the command toolbar after its first display error An error raised while the toolbar redraws during a command, most likely from get_bottom_toolbar(), went to asyncio's default handler, which logs a traceback to stderr. That write passes through the stdout proxy, whose redraw calls the failing callback again, so one faulty callback printed hundreds of tracebacks per second for as long as the command ran. Handle errors on the display's event loop by stopping the display, which reports the error once through the path that already existed for a display that stops on its own. A display which has failed is not restarted for the rest of the command, and a display which cannot come back after a pause is reported instead of raised: that exception would otherwise fail the command which paused it, or escape cmdloop() from command finalization. --- cmd2/command_toolbar.py | 46 ++++++++++++++++-- tests/test_command_toolbar.py | 89 +++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 3 deletions(-) diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index d42630317..5e38169ee 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -1,5 +1,6 @@ """Internal support for displaying a toolbar during synchronous commands.""" +import asyncio import codecs import contextlib import contextvars @@ -169,6 +170,9 @@ def __init__(self, cmd: "Cmd") -> None: self._proxy: StdoutProxy | None = None self._lock = threading.RLock() self._pausing = False + # Set once the display has stopped after an error. It is reported once and not + # restarted, so a callback which keeps failing cannot repeat that at every pause. + self._failed = False session = cmd.main_session self.app = session.app @@ -267,6 +271,8 @@ def _restore_stream(obj: Any, name: str, stream: ToolbarStream) -> None: setattr(obj, name, stream.original) def _resume(self) -> None: + if self._failed: + return self._ready.clear() self._error = None stack = self._display_stack = contextlib.ExitStack() @@ -279,7 +285,7 @@ def _resume(self) -> None: def run() -> None: try: - self.app.run(handle_sigint=False, set_exception_handler=False) + self.app.run(pre_run=self._handle_loop_errors, handle_sigint=False, set_exception_handler=False) except EOFError: pass except BaseException as exc: # noqa: BLE001 @@ -302,6 +308,27 @@ def run() -> None: for stream in self._streams: stream.proxy = proxy + def _handle_loop_errors(self) -> None: + """Stop the display on the first unhandled error in its event loop. + + prompt-toolkit's own handler prints the traceback and waits for Enter, which would + hold up a command that is still running. With no handler at all, asyncio logs each + error to stderr. That write goes through the stdout proxy, whose redraw of the + toolbar calls a failing ``get_bottom_toolbar()`` again, so a single faulty callback + buries the command's output in tracebacks for as long as the command runs. + """ + asyncio.get_running_loop().set_exception_handler(self._loop_error) + + def _loop_error(self, loop: asyncio.AbstractEventLoop, context: dict[str, Any]) -> None: # noqa: ARG002 + """Record the first error for the command thread and end the display. + + ``_resume()`` raises it if the display has not finished starting. After that, + ``_app_exited()`` reports it once. + """ + if self._error is None: + self._error = context.get("exception") or RuntimeError(context.get("message", "unknown error")) + self._exit() + def _app_exited(self) -> None: """Give the terminal back to the streams when the display stops on its own. @@ -324,7 +351,13 @@ def _app_exited(self) -> None: # A proxy exists only once _resume() has handed startup failures to the command # thread, so reporting here does not duplicate the exception it raises. if started and self._error is not None: - self.cmd.perror(f"Bottom toolbar stopped after an error: {self._error!r}") + self._report_failure(self._error) + + def _report_failure(self, error: BaseException) -> None: + """Say once that the command carries on without its toolbar.""" + if not self._failed: + self._failed = True + self.cmd.perror(f"Bottom toolbar stopped after an error: {error!r}") def _exit(self) -> None: """Stop the display unless it has already stopped on its own. @@ -505,4 +538,11 @@ def suspend(self) -> Generator[None]: yield finally: with self.cmd.sigint_protection: - self._resume() + try: + self._resume() + except Exception as exc: # noqa: BLE001 + # The toolbar is cosmetic. A display that cannot come back must not fail + # the command which paused it, nor escape cmdloop() from command + # finalization. The command carries on without one, and stop() still + # returns the borrowed application to the main prompt. + self._report_failure(exc) diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index 43c6b9c18..80e419b48 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -665,6 +665,95 @@ def test_command_toolbar_failure_after_startup_is_reported(toolbar_app, capsys) assert "after failure\n" in output.getvalue() +def test_command_toolbar_stops_once_after_a_callback_error(toolbar_app, capsys) -> None: + """An error from the toolbar callback ends the display instead of being logged at every redraw.""" + app, _, output = toolbar_app + broken = threading.Event() + failures = [] + + def toolbar(): + if broken.is_set(): + failures.append(None) + raise ValueError("broken toolbar") + return "STATUS" + + app.main_session.bottom_toolbar = toolbar + with app._command_toolbar_context(): + display = app._command_toolbar + thread = display._thread + broken.set() + thread.join(timeout=2) + assert not thread.is_alive() + + # Output must still reach the terminal rather than a proxy nothing is draining. + assert all(stream.proxy is None for stream in display._streams) + app.poutput("after failure") + + # A display which failed is not restarted, so a later pause neither calls the + # callback again nor repeats the report. + seen = len(failures) + with app.suspend_bottom_toolbar(): + pass + assert display._thread is None + assert not display.is_active + assert len(failures) == seen + + err = capsys.readouterr().err + assert err.count("Bottom toolbar stopped after an error") == 1 + assert "broken toolbar" in err + assert "Traceback" not in err + assert "after failure\n" in output.getvalue() + assert app.main_session.app.layout is app.main_session.layout + + # The next command starts a display of its own. + broken.clear() + with app._command_toolbar_context(): + assert app._command_toolbar.is_active + + +def test_command_toolbar_reports_a_loop_error_without_an_exception(toolbar_app, capsys) -> None: + app, _, _ = toolbar_app + with app._command_toolbar_context(): + display = app._command_toolbar + thread = display._thread + loop = display.app.loop + loop.call_soon_threadsafe(loop.call_exception_handler, {"message": "loop misbehaved"}) + thread.join(timeout=2) + assert not thread.is_alive() + + assert "loop misbehaved" in capsys.readouterr().err + + +def test_command_toolbar_failing_to_resume_does_not_fail_the_command(toolbar_app, capsys) -> None: + """A display which cannot come back after a pause is reported, not raised into the command.""" + app, _, output = toolbar_app + broken = threading.Event() + + def toolbar(): + if broken.is_set(): + raise ValueError("broken toolbar") + return "STATUS" + + app.main_session.bottom_toolbar = toolbar + with app._command_toolbar_context(): + display = app._command_toolbar + with app.suspend_bottom_toolbar(): + broken.set() + assert not display.is_active + app.poutput("after failure") + + # Command finalization pauses the display as well, at the end of every command. + assert app._run_cmdfinalization_hooks(False, None) is False + + err = capsys.readouterr().err + assert err.count("Bottom toolbar stopped after an error") == 1 + assert "broken toolbar" in err + assert "after failure\n" in output.getvalue() + assert app.main_session.app.layout is app.main_session.layout + assert app.stdout is output + assert app._command_toolbar is None + + def test_command_toolbar_input_eof_is_not_reported(toolbar_app, capsys) -> None: app, pipe, output = toolbar_app From b511661d215515fd88c16baf718617c6e2e42214 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:17:16 -0500 Subject: [PATCH 26/35] Document the toolbar callback's locking and failure rules Pausing the toolbar waits for the refresh in progress, so a command holding a lock which get_bottom_toolbar() acquires must not prompt, page or run a subprocess while it holds it. Say so where the documentation recommends a lock. Also note at the key_bindings swap that prompt-toolkit caches merged bindings per focused window and does not see the attribute being reassigned. --- cmd2/cmd2.py | 7 ++++++- cmd2/command_toolbar.py | 5 +++++ docs/features/prompt.md | 12 ++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index 397fbf5ad..b99ea79e3 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -2129,9 +2129,14 @@ def get_bottom_toolbar(self) -> AnyFormattedText: or even a real-time clock. During command execution this callback runs in a background UI thread. Protect shared - state with a lock when necessary. The built-in pager shares this toolbar. It is suspended + state with a lock when necessary, but never hold that lock in a command while it prompts + for input, pages output, runs a shell command, or otherwise pauses the toolbar: pausing + waits for this callback to return. The built-in pager shares this toolbar. It is suspended while another prompt, external pager, or interactive shell owns the terminal. + If this callback raises an exception during a command, the error is reported once and the + command carries on without the toolbar. + :return: Content to populate the bottom toolbar. """ return None diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index 5e38169ee..bac89cef3 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -276,6 +276,11 @@ def _resume(self) -> None: self._ready.clear() self._error = None stack = self._display_stack = contextlib.ExitStack() + # Replace key_bindings only together with a layout whose focused window prompt-toolkit + # has not seen with other bindings. It caches the bindings it merges for each focused + # window, application bindings included, and does not notice this attribute being + # reassigned, so reusing a window with different bindings would keep the old ones. + # This layout, the main prompt's, and each pager's all have windows of their own. for name, value in (("layout", self._layout), ("key_bindings", self._bindings), ("erase_when_done", True)): stack.callback(setattr, self.app, name, getattr(self.app, name)) setattr(self.app, name, value) diff --git a/docs/features/prompt.md b/docs/features/prompt.md index 31f036675..6f97bb3a6 100644 --- a/docs/features/prompt.md +++ b/docs/features/prompt.md @@ -132,6 +132,18 @@ while the command prints output. During command execution, `get_bottom_toolbar()` runs in a background UI thread. Keep the callback fast and use a lock when reading state that a command or another thread modifies. +!!! warning + + A command must not hold a lock which `get_bottom_toolbar()` acquires while it calls anything + that pauses the toolbar: `read_input()`, `read_secret()`, `select()`, `ppaged()`, `do_shell()`, + `run_editor()`, an embedded Python shell, or `suspend_bottom_toolbar()`. To pause the toolbar, + `cmd2` waits for the refresh in progress to finish. A callback blocked on a lock the command is + holding never finishes, so the application hangs. Copy what you need while holding the lock, + release it, and then prompt or page. + +If `get_bottom_toolbar()` raises an exception while a command is running, `cmd2` reports the error +once and the command carries on without the toolbar. + !!! tip If you want to show more realtime status information than fits in a toolbar, you can run your From fb4a2d9fd3cfe958964a211fcd43c2ca75616ce2 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:37:59 -0500 Subject: [PATCH 27/35] Keep unfinished lines of command output above the toolbar prompt-toolkit draws the toolbar starting at the cursor, and erases it from the first column of that row before printing anything more above it. Output which stopped in the middle of a line was erased along with the toolbar: a line written as 'Working...' and finished later lost its first part, a progress line redrawn with carriage returns vanished once it was complete, and a last line without a newline was lost whenever the display redrew before it stopped. End an unfinished line on the screen so the toolbar always starts on a row of its own, and when more output arrives move back up and write the unfinished line again before it, which puts the cursor and any style it set where they were. A line unfinished when the display stops stays ended, so the next prompt starts on a new line, as it does with reserved rendering. --- CHANGELOG.md | 3 +- cmd2/command_toolbar.py | 82 +++++++++++++++++++++++++++-- docs/features/prompt.md | 5 ++ tests/test_command_toolbar.py | 98 +++++++++++++++++++++++++++++++++++ 4 files changed, 182 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3e5159dc..bc89d83f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ - Enhancements - `enable_bottom_toolbar=True` now keeps the toolbar visible and refreshing during command - execution + execution. Command output appears above it. A line of output left unfinished when a command + ends is ended there, so the next prompt starts on a new line while the toolbar is enabled. - `Cmd.read_input()` and `Cmd.read_secret()` now keep the bottom toolbar visible while they wait for input, refreshing at the same `refresh_interval` as the main prompt, instead of the toolbar disappearing for the duration of the nested prompt. `Cmd.select()` is unchanged, since diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index bac89cef3..c26920d06 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -6,6 +6,7 @@ import contextvars import functools import os +import re import signal import sys import threading @@ -24,13 +25,17 @@ from prompt_toolkit.layout import HSplit, Layout, Window from prompt_toolkit.layout.containers import ConditionalContainer from prompt_toolkit.patch_stdout import StdoutProxy -from prompt_toolkit.utils import suspend_to_background_supported +from prompt_toolkit.utils import get_cwidth, suspend_to_background_supported from .pager import Pager, output_fits if TYPE_CHECKING: from .cmd2 import Cmd +# Control sequences, operating system commands, and two-character escapes: text which takes +# no room on the screen. +_ESCAPE_RE = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|\][^\x07\x1b]*(?:\x07|\x1b\\)|[@-Z\\-_])") + _F = TypeVar("_F", bound=Callable[..., Any]) _R = TypeVar("_R") @@ -91,7 +96,22 @@ def pipe_target(stream: Any) -> Any: class _ContextStdoutProxy(StdoutProxy): - """Keep stdout's flush worker in the toolbar's isolated application session.""" + """Print above the toolbar from its isolated application session, keeping unfinished lines. + + prompt-toolkit draws the toolbar starting at the cursor, and erases it from the first + column of that row before printing anything more. Output which stopped in the middle of + a line, such as ``Working...`` or a progress line ending in a carriage return, would be + erased with it. So a line left unfinished is ended on the screen, which gives the toolbar + a row of its own, and it is continued from where it stopped when more output arrives. + """ + + def __init__(self) -> None: + # The line which output last left unfinished, exactly as written. Set before the + # base class starts the thread which reads it. + self._unfinished = "" + # The worker already combines queued writes. A batching sleep would also delay + # close(), which runs at each command finalization boundary. + super().__init__(raw=True, sleep_between_writes=0) def _start_write_thread(self) -> threading.Thread: context = contextvars.copy_context() @@ -99,6 +119,60 @@ def _start_write_thread(self) -> threading.Thread: thread.start() return thread + def _write_and_flush(self, loop: asyncio.AbstractEventLoop | None, text: str) -> None: + """Write text above the toolbar as the base class does, continuing an unfinished line.""" + + def write_and_flush() -> None: + output = self._output + # See the base class: Windows resets autowrap at each flush. + output.enable_autowrap() + if self._unfinished: + # The cursor is at the start of the row below the unfinished line. Go back + # to the row its last carriage return left it on, and write the rest of + # it again to put the cursor, and any style it set, where they were. + output.cursor_up(self._rows(self._unfinished)) + output.write_raw(self._unfinished) + output.write_raw(text) + line = self._last_line(self._unfinished + text) + if line: + output.write_raw("\r\n") + size = output.get_size() + # A line whose start has scrolled off the screen can't be found again, so it + # stays ended where it is. + self._unfinished = line if self._width(line) <= size.rows * size.columns else "" + output.flush() + + if loop is None: + write_and_flush() + else: + loop.call_soon_threadsafe(lambda: run_in_terminal(write_and_flush, in_executor=False)) + + @staticmethod + def _last_line(text: str) -> str: + """Return what must be written again to continue the line which text leaves unfinished. + + :param text: everything written since the last finished line + :return: the unfinished line from its last carriage return, or "" if the last line is + finished + """ + line = text.rpartition("\n")[2] + if not line.strip("\r"): + # Nothing has been written on this row, wherever the cursor returned to. + return "" + # A carriage return went back to the start of the row the cursor is on, so nothing + # before it is needed to find that place again. Dropping it also keeps a progress + # line which is redrawn thousands of times from growing without bound. + return line[max(0, line.rfind("\r")) :] + + def _rows(self, line: str) -> int: + """Return how many rows below its start the cursor was left after line was ended.""" + columns = max(1, self._output.get_size().columns) + return max(1, -(-self._width(line) // columns)) + + @staticmethod + def _width(line: str) -> int: + return get_cwidth(_ESCAPE_RE.sub("", line)) + class ToolbarStream: """Keep a stable stream identity across suspensions and cmd2 redirections.""" @@ -305,9 +379,7 @@ def run() -> None: self._ready.wait() if self._error is not None: raise self._error - # The worker already combines queued writes. A batching sleep would also - # delay close(), which runs at each command finalization boundary. - proxy = _ContextStdoutProxy(raw=True, sleep_between_writes=0) + proxy = _ContextStdoutProxy() with self._lock: self._proxy = proxy for stream in self._streams: diff --git a/docs/features/prompt.md b/docs/features/prompt.md index 6f97bb3a6..318e37138 100644 --- a/docs/features/prompt.md +++ b/docs/features/prompt.md @@ -182,6 +182,11 @@ terminal is in raw mode, so the kernel generates no signals from keystrokes. cmd to its own process group, which reaches a subprocess your command started and waits on, but `Ctrl-\` (`SIGQUIT`) does nothing, exactly as at the main prompt. +Output which stops in the middle of a line, such as `Working...` or a progress line redrawn with +carriage returns, stays on the screen and is continued by the output which follows it. A line still +unfinished when the command ends, or when the toolbar is hidden for one of the operations above, is +ended there, so whatever comes next starts on a new line. + The command toolbar is used by the interactive command loop, including startup commands and scripts launched from that loop. It is disabled for non-interactive input. Calls to `onecmd_plus_hooks()` outside the command loop do not start a toolbar. diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index 80e419b48..e1acddefd 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -320,6 +320,104 @@ def popen(*args, **kwargs): assert running == [False] +class CursorOutput(RecordingOutput): + """Record moving the cursor up along with the text written.""" + + def cursor_up(self, amount: int) -> None: + self.stdout.write(f"") + + +@pytest.mark.parametrize( + ("writes", "expected"), + [ + pytest.param(["one\n", "two\nthree\n"], "one\ntwo\nthree\n", id="finished lines are written as they are"), + pytest.param( + ["Working...", " finished\n"], + "Working...\r\nWorking... finished\n", + id="an unfinished line is ended for the toolbar and continued", + ), + pytest.param( + ["\rprogress 10%", "\rprogress 20%", "\n"], + "\rprogress 10%\r\n\rprogress 10%\rprogress 20%\r\n\rprogress 20%\n", + id="a redrawn progress line is continued from its last carriage return", + ), + pytest.param( + ["progress 10%\r", "progress 20%\r", "done\n"], + "progress 10%\r\r\n\rprogress 20%\r\r\n\rdone\n", + id="a line ending in a carriage return is still unfinished", + ), + pytest.param(["one\n\r", "two\n"], "one\n\rtwo\n", id="a carriage return on an empty row leaves nothing unfinished"), + pytest.param( + ["x" * 200, "END\n"], + "x" * 200 + "\r\n" + "x" * 200 + "END\n", + id="a wrapped line is continued from the row it started on", + ), + pytest.param( + ["y" * 80, "Z\n"], + "y" * 80 + "\r\n" + "y" * 80 + "Z\n", + id="a line as wide as the screen has not wrapped yet", + ), + pytest.param( + ["ζΌ’" * 41, "END\n"], + "ζΌ’" * 41 + "\r\n" + "ζΌ’" * 41 + "END\n", + id="wide characters take two columns", + ), + pytest.param( + ["\x1b[31mred\x1b]8;;http://example.com\x07", " still\x1b[0m\n"], + "\x1b[31mred\x1b]8;;http://example.com\x07\r\n\x1b[31mred\x1b]8;;http://example.com\x07 still\x1b[0m\n", + id="escape sequences take no room and are written again", + ), + pytest.param( + ["z" * (24 * 80 + 1), "END\n"], + "z" * (24 * 80 + 1) + "\r\nEND\n", + id="a line longer than the screen stays ended", + ), + ], +) +def test_stdout_proxy_continues_unfinished_lines(writes, expected) -> None: + """Output which stops in the middle of a line survives the toolbar being drawn below it.""" + stream = Terminal() + with create_pipe_input() as pipe, create_app_session(input=pipe, output=CursorOutput(stream)): + proxy = command_toolbar._ContextStdoutProxy() + try: + # No application is running, so each write reaches the output before the next. + for text in writes: + proxy._write_and_flush(None, text) + finally: + proxy.close() + + assert stream.getvalue() == expected + + +@pytest.mark.parametrize("flushed", [True, False]) +def test_command_toolbar_keeps_an_unfinished_line(toolbar_app, flushed) -> None: + """An unfinished line is continued by later output, and ended when the toolbar stops.""" + app, _, output = toolbar_app + terminal = CursorOutput(output) + app.main_session.app.output = terminal + with app._command_toolbar_context(): + proxy = app._command_toolbar._proxy + app.stdout.write("Working...") + if flushed: + app.stdout.flush() + # Let the display print this much before any more is written. + deadline = time.monotonic() + 2 + while "Working..." not in output.getvalue() and time.monotonic() < deadline: + time.sleep(0.01) + app.stdout.write(" finished\n") + app.stdout.write("last line") + assert proxy.closed + + text = output.getvalue() + if flushed: + assert "Working...\r\n" in text + assert "Working... finished\n" in text + else: + assert "Working... finished\n" in text + # The prompt which follows starts on a row of its own. + assert "last line\r\n" in text + + class CprOutput(RecordingOutput): """A terminal that asks for cursor position reports and never answers them.""" From 809a1d55d5360ef232b52b782a8bd652ce9b8fed Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:58:41 -0500 Subject: [PATCH 28/35] Report the error which stopped the toolbar, not one from its last frame Stopping the display after an error draws a last frame and fires after_render again. A handler which fails there replaced the error already recorded, so the report named a consequence instead of the cause. That is how a race in the pager navigation test surfaced on CI. Its after_render check read the terminal's size after another thread had resized it, so a frame drawn at the old size was judged against the new one. asyncio used to log and ignore that assertion. Now it stops the display, and the check failed again on the last frame with an unrelated AttributeError. Judge each frame against the size it was drawn at. --- cmd2/command_toolbar.py | 7 +++++-- tests/test_command_toolbar.py | 25 +++++++++++++++++++++++++ tests/test_pager.py | 4 +++- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index c26920d06..b18b75892 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -368,8 +368,11 @@ def run() -> None: except EOFError: pass except BaseException as exc: # noqa: BLE001 - # Propagate startup/render failures to the command thread. - self._error = exc + # Propagate startup/render failures to the command thread. Stopping after an + # error in the loop draws a last frame, and a failure there is only a + # consequence of the error already recorded. + if self._error is None: + self._error = exc finally: self._ready.set() self._app_exited() diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index e1acddefd..727875ff7 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -809,6 +809,31 @@ def toolbar(): assert app._command_toolbar.is_active +def test_command_toolbar_reports_the_error_which_stopped_it(toolbar_app, capsys) -> None: + """Stopping draws a last frame. A failure there must not replace the error which caused the stop.""" + app, _, _ = toolbar_app + frames = [] + + def fail(ui): + frames.append(None) + raise ValueError(f"error {len(frames)}") + + with app._command_toolbar_context(): + display = app._command_toolbar + thread = display._thread + display.app.after_render += fail + try: + thread.join(timeout=2) + assert not thread.is_alive() + finally: + display.app.after_render -= fail + + err = capsys.readouterr().err + assert len(frames) == 2 + assert "ValueError('error 1')" in err + assert "error 2" not in err + + def test_command_toolbar_reports_a_loop_error_without_an_exception(toolbar_app, capsys) -> None: app, _, _ = toolbar_app with app._command_toolbar_context(): diff --git a/tests/test_pager.py b/tests/test_pager.py index a2a4283ba..b53aff4de 100644 --- a/tests/test_pager.py +++ b/tests/test_pager.py @@ -66,7 +66,9 @@ def observe(ui): # Check the rendered frame, not the stream of incremental terminal # writes, to verify both toolbar rows survive navigation and resizing. screen = ui.renderer._last_screen - size = ui.output.get_size() + # Use the size this frame was drawn at. The terminal may have been resized since, + # and an error raised here stops the display. + size = ui.renderer._last_size bottom = "".join(screen.data_buffer[size.rows - 1][x].char for x in range(size.columns)) assert bottom.startswith("STATUS TWO") entered.set() From ac0cc0bc93945f927a0d017ff529e239e49db8ff Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Sat, 14 Nov 2026 19:15:40 -0500 Subject: [PATCH 29/35] Count the rows of an unfinished line by the width Windows wraps at prompt-toolkit's Windows outputs report one column fewer than the console has, to keep what it draws out of the last column. Raw command output is wrapped by the console itself, at the full width of its screen buffer. Rows were counted by the reported size, so an unfinished line which filled the last column was counted as one row longer than it was: continuing it went up one row too many, wrote over the line above, and left the rest of the output on top of the line's own last row. That takes a line exactly as wide as the window, or within one column short of two widths, two short of three, and so on. The 200 columns of the manual check's wrap command fail in a window 100 columns wide and pass in one of 120, which is why the default window size showed nothing. Use the width of the console's screen buffer when the output has one. That is also right for a console whose buffer is wider than its window, which wraps at the buffer's width. --- cmd2/command_toolbar.py | 15 ++++++++++++-- tests/test_command_toolbar.py | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index b18b75892..86cdda7da 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -166,8 +166,19 @@ def _last_line(text: str) -> str: def _rows(self, line: str) -> int: """Return how many rows below its start the cursor was left after line was ended.""" - columns = max(1, self._output.get_size().columns) - return max(1, -(-self._width(line) // columns)) + return max(1, -(-self._width(line) // self._wrap_columns())) + + def _wrap_columns(self) -> int: + """Return how many columns the terminal fits on a row before it wraps a line. + + prompt-toolkit's Windows outputs report one column fewer than the console has, which + keeps what prompt-toolkit draws out of the last column. Text written raw is wrapped by + the console itself, at the width of its screen buffer. + """ + screen_buffer_info = getattr(self._output, "get_win32_screen_buffer_info", None) + if screen_buffer_info is not None: + return max(1, int(screen_buffer_info().dwSize.X)) + return max(1, self._output.get_size().columns) @staticmethod def _width(line: str) -> int: diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index 727875ff7..8fae42d0c 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -389,6 +389,44 @@ def test_stdout_proxy_continues_unfinished_lines(writes, expected) -> None: assert stream.getvalue() == expected +class WindowsConsoleOutput(CursorOutput): + """Report a size the way prompt-toolkit's Windows outputs do, one column short of the console's.""" + + def __init__(self, stream: Terminal) -> None: + super().__init__(stream) + self.size = Size(rows=24, columns=79) + + def get_win32_screen_buffer_info(self) -> SimpleNamespace: + return SimpleNamespace(dwSize=SimpleNamespace(X=80, Y=24)) + + +@pytest.mark.parametrize( + ("width", "rows"), + [ + pytest.param(80, 1, id="a line as wide as the console"), + pytest.param(159, 2, id="a line one column short of two rows"), + pytest.param(160, 2, id="a line which fills two rows"), + pytest.param(161, 3, id="a line which reaches a third row"), + ], +) +def test_stdout_proxy_counts_rows_by_the_width_a_windows_console_wraps_at(width, rows) -> None: + """A line which fills the last column of a Windows console is continued from its own row. + + prompt-toolkit reports such a console as one column narrower than the width it wraps at. + Counting rows by that size went up one row too many, and wrote over the line above. + """ + stream = Terminal() + with create_pipe_input() as pipe, create_app_session(input=pipe, output=WindowsConsoleOutput(stream)): + proxy = command_toolbar._ContextStdoutProxy() + try: + proxy._write_and_flush(None, "x" * width) + proxy._write_and_flush(None, "END\n") + finally: + proxy.close() + + assert stream.getvalue() == "x" * width + f"\r\n" + "x" * width + "END\n" + + @pytest.mark.parametrize("flushed", [True, False]) def test_command_toolbar_keeps_an_unfinished_line(toolbar_app, flushed) -> None: """An unfinished line is continued by later output, and ended when the toolbar stops.""" From cbfb5660e1f3bf6e9158ac9e051cd19518c0f651 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Sat, 14 Nov 2026 19:27:10 -0500 Subject: [PATCH 30/35] Leave an unfinished line ended once its first row has scrolled away An unfinished line was continued whenever its width fitted in the rows and columns of the screen. A line which reaches the last row has already lost its first row by then: ending it scrolls the screen, so going back up stops one row short of where the line began, and writing the line again from there shows its first row twice. A toolbar of several rows scrolls the screen sooner, by a row for each row it has beyond the first, so a bound which allows for one row is not enough. Continue a line only while it and the toolbar below it fit on the screen together. The display tells the proxy how many rows its toolbar took when it was last drawn. A line too long for that stays ended, as one longer than the screen already did, and the output which follows starts on the next row. --- cmd2/command_toolbar.py | 23 +++++++++++++++----- tests/test_command_toolbar.py | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index 86cdda7da..7dc3faa1d 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -105,10 +105,15 @@ class _ContextStdoutProxy(StdoutProxy): a row of its own, and it is continued from where it stopped when more output arrives. """ - def __init__(self) -> None: + def __init__(self, toolbar_rows: Callable[[], int] = lambda: 1) -> None: + """Start printing above a toolbar. + + :param toolbar_rows: returns how many rows the toolbar takes below the output + """ # The line which output last left unfinished, exactly as written. Set before the # base class starts the thread which reads it. self._unfinished = "" + self._toolbar_rows = toolbar_rows # The worker already combines queued writes. A batching sleep would also delay # close(), which runs at each command finalization boundary. super().__init__(raw=True, sleep_between_writes=0) @@ -136,10 +141,12 @@ def write_and_flush() -> None: line = self._last_line(self._unfinished + text) if line: output.write_raw("\r\n") - size = output.get_size() # A line whose start has scrolled off the screen can't be found again, so it - # stays ended where it is. - self._unfinished = line if self._width(line) <= size.rows * size.columns else "" + # stays ended where it is. Its first row scrolls off once the screen has no + # room for the whole line and the toolbar below it. A row is needed below the + # line even with no toolbar, for the cursor which ended it. + room = output.get_size().rows - max(1, self._toolbar_rows()) + self._unfinished = line if self._rows(line) <= room else "" output.flush() if loop is None: @@ -279,6 +286,7 @@ def __init__(self, cmd: "Cmd") -> None: and self.toolbar.content.style == "class:bottom-toolbar" ): raise RuntimeError("Cannot locate PromptSession bottom toolbar") + self._toolbar_window = self.toolbar.content self._layout = Layout(HSplit([Window(height=0), Window(), self.toolbar])) self._display_stack: contextlib.ExitStack | None = None bindings = KeyBindings() @@ -328,6 +336,11 @@ def _suspend_to_background(suspend_group: bool = True) -> None: def _after_render(self, app: Application[str]) -> None: # noqa: ARG002 self._ready.set() + def _toolbar_rows(self) -> int: + """Return how many rows the toolbar took when it was last drawn.""" + info = self._toolbar_window.render_info + return info.window_height if info is not None else 0 + def start(self) -> None: """Start rendering and protect terminal output.""" stack = contextlib.ExitStack() @@ -393,7 +406,7 @@ def run() -> None: self._ready.wait() if self._error is not None: raise self._error - proxy = _ContextStdoutProxy() + proxy = _ContextStdoutProxy(self._toolbar_rows) with self._lock: self._proxy = proxy for stream in self._streams: diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index 8fae42d0c..8b00adfde 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -372,6 +372,16 @@ def cursor_up(self, amount: int) -> None: "z" * (24 * 80 + 1) + "\r\nEND\n", id="a line longer than the screen stays ended", ), + pytest.param( + ["z" * (23 * 80 + 1), "END\n"], + "z" * (23 * 80 + 1) + "\r\nEND\n", + id="a line which reaches the last row stays ended", + ), + pytest.param( + ["z" * (23 * 80), "END\n"], + "z" * (23 * 80) + "\r\n" + "z" * (23 * 80) + "END\n", + id="a line which stops above the last row is continued", + ), ], ) def test_stdout_proxy_continues_unfinished_lines(writes, expected) -> None: @@ -389,6 +399,37 @@ def test_stdout_proxy_continues_unfinished_lines(writes, expected) -> None: assert stream.getvalue() == expected +@pytest.mark.parametrize( + ("width", "expected"), + [ + pytest.param(22 * 80, "\r\n" + "z" * (22 * 80) + "END\n", id="continued while the toolbar fits below it"), + pytest.param(22 * 80 + 1, "\r\nEND\n", id="stays ended once the toolbar would scroll its first row off"), + ], +) +def test_stdout_proxy_leaves_room_for_every_row_of_the_toolbar(width, expected) -> None: + """A toolbar of several rows scrolls the screen sooner than one row does.""" + stream = Terminal() + with create_pipe_input() as pipe, create_app_session(input=pipe, output=CursorOutput(stream)): + proxy = command_toolbar._ContextStdoutProxy(lambda: 2) + try: + proxy._write_and_flush(None, "z" * width) + proxy._write_and_flush(None, "END\n") + finally: + proxy.close() + + assert stream.getvalue() == "z" * width + expected + + +def test_command_toolbar_tells_the_proxy_how_many_rows_it_takes(toolbar_app) -> None: + app, _, _ = toolbar_app + app.main_session.bottom_toolbar = "one\ntwo" + # Nothing has been drawn yet. + assert command_toolbar.CommandToolbar(app)._toolbar_rows() == 0 + with app._command_toolbar_context(): + assert app._command_toolbar._toolbar_rows() == 2 + assert app._command_toolbar._proxy._toolbar_rows() == 2 + + class WindowsConsoleOutput(CursorOutput): """Report a size the way prompt-toolkit's Windows outputs do, one column short of the console's.""" From 939ce21ec06d6eb1ec3032bbb3f1cb64afd33fb8 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Sat, 14 Nov 2026 20:03:46 -0500 Subject: [PATCH 31/35] Count the row a legacy Windows console has already wrapped to A terminal which understands escape sequences leaves the cursor in the last column when a row has just been filled, and wraps at the next character. A legacy Windows console moves to the next row at once. Ending an unfinished line which exactly fills its rows therefore leaves the cursor a row lower there. Rows were counted as for a terminal which waits, so continuing such a line went up one row too few and wrote it again a row below itself, which showed its last row twice. Counting rows by the size prompt-toolkit reports, one column short, happened to give the right answer for these lines on a legacy console. Counting by the width the console wraps at fixed every other console and broke this one. A legacy console gets prompt-toolkit's Win32Output, which unlike its other Windows outputs has no vt100 output. Count one more row for a line which has just filled a row when the output is that one. --- cmd2/command_toolbar.py | 17 +++++++++++++- tests/test_command_toolbar.py | 44 +++++++++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index 7dc3faa1d..ff2ac93cc 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -173,7 +173,22 @@ def _last_line(text: str) -> str: def _rows(self, line: str) -> int: """Return how many rows below its start the cursor was left after line was ended.""" - return max(1, -(-self._width(line) // self._wrap_columns())) + width = self._width(line) + columns = self._wrap_columns() + if self._wraps_at_once(): + # A line which has just filled a row has put the cursor on the next one. + return width // columns + 1 + return max(1, -(-width // columns)) + + def _wraps_at_once(self) -> bool: + """Return whether the terminal moves to the next row as soon as a row is full. + + A terminal which understands escape sequences waits for the next character. A legacy + Windows console does not wait. It gets prompt-toolkit's Win32Output which, unlike its + other Windows outputs, has no vt100 output to write escape sequences with. + """ + output = self._output + return hasattr(output, "get_win32_screen_buffer_info") and not hasattr(output, "vt100_output") def _wrap_columns(self) -> int: """Return how many columns the terminal fits on a row before it wraps a line. diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index 8b00adfde..c15c53537 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -430,8 +430,11 @@ def test_command_toolbar_tells_the_proxy_how_many_rows_it_takes(toolbar_app) -> assert app._command_toolbar._proxy._toolbar_rows() == 2 -class WindowsConsoleOutput(CursorOutput): - """Report a size the way prompt-toolkit's Windows outputs do, one column short of the console's.""" +class LegacyConsoleOutput(CursorOutput): + """Report a size the way prompt-toolkit's Windows outputs do, one column short of the console's. + + Like prompt-toolkit's Win32Output, which a legacy console gets, it has no vt100 output. + """ def __init__(self, stream: Terminal) -> None: super().__init__(stream) @@ -441,6 +444,14 @@ def get_win32_screen_buffer_info(self) -> SimpleNamespace: return SimpleNamespace(dwSize=SimpleNamespace(X=80, Y=24)) +class WindowsConsoleOutput(LegacyConsoleOutput): + """A console which understands escape sequences, as prompt-toolkit's Windows10_Output writes to.""" + + def __init__(self, stream: Terminal) -> None: + super().__init__(stream) + self.vt100_output = self + + @pytest.mark.parametrize( ("width", "rows"), [ @@ -468,6 +479,35 @@ def test_stdout_proxy_counts_rows_by_the_width_a_windows_console_wraps_at(width, assert stream.getvalue() == "x" * width + f"\r\n" + "x" * width + "END\n" +@pytest.mark.parametrize( + ("width", "rows"), + [ + pytest.param(79, 1, id="a line one column short of the console's width"), + pytest.param(80, 2, id="a line as wide as the console has already wrapped"), + pytest.param(81, 2, id="a line which reaches a second row"), + pytest.param(159, 2, id="a line one column short of two rows"), + pytest.param(160, 3, id="a line which fills two rows has already wrapped"), + ], +) +def test_stdout_proxy_counts_the_row_a_legacy_windows_console_has_wrapped_to(width, rows) -> None: + """A legacy console moves to the next row as soon as a row is full, not at the next character. + + Ending a line which exactly fills its rows therefore leaves the cursor a row lower than + on a terminal which waits. Going up one row too few wrote the line again a row below + itself, which showed its last row twice. + """ + stream = Terminal() + with create_pipe_input() as pipe, create_app_session(input=pipe, output=LegacyConsoleOutput(stream)): + proxy = command_toolbar._ContextStdoutProxy() + try: + proxy._write_and_flush(None, "x" * width) + proxy._write_and_flush(None, "END\n") + finally: + proxy.close() + + assert stream.getvalue() == "x" * width + f"\r\n" + "x" * width + "END\n" + + @pytest.mark.parametrize("flushed", [True, False]) def test_command_toolbar_keeps_an_unfinished_line(toolbar_app, flushed) -> None: """An unfinished line is continued by later output, and ended when the toolbar stops.""" From 2f4b5ddb18ccf6e51ff21436223b9bb2f5d22acd Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Sat, 14 Nov 2026 20:17:05 -0500 Subject: [PATCH 32/35] Remember the toolbar's rows until it has been drawn for the command An unfinished line is continued only while it and the toolbar fit on the screen together, and the display reported the toolbar's rows from the frame it last drew. Each run of the application forgets what it drew, and prompt-toolkit leaves the toolbar out until the terminal has reported where its cursor is. A command which writes at once does so before that report arrives, so the toolbar counted for no rows and a toolbar of two rows was allowed for as one: a line one row too tall for it was continued, and its first row was shown twice. Windows knows the height of its console without asking, so this showed only on terminals which answer a cursor position request. Keep the rows from the last frame which drew the toolbar, starting with the main prompt's, which are the rows it is about to take again. --- cmd2/command_toolbar.py | 16 ++++++++++++++-- tests/test_command_toolbar.py | 27 +++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index ff2ac93cc..0d59e3e9e 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -302,6 +302,12 @@ def __init__(self, cmd: "Cmd") -> None: ): raise RuntimeError("Cannot locate PromptSession bottom toolbar") self._toolbar_window = self.toolbar.content + # The rows the toolbar took when it was last drawn, to begin with by the main prompt. + # Each run of the application forgets what it drew, and prompt-toolkit leaves the + # toolbar out until the terminal has reported where its cursor is, which can be after + # a command's first output. The toolbar is about to take these rows again. + self._toolbar_height = 0 + self._note_toolbar_height() self._layout = Layout(HSplit([Window(height=0), Window(), self.toolbar])) self._display_stack: contextlib.ExitStack | None = None bindings = KeyBindings() @@ -349,12 +355,18 @@ def _suspend_to_background(suspend_group: bool = True) -> None: run_in_terminal(functools.partial(suspend_process_group, suspend_group)) def _after_render(self, app: Application[str]) -> None: # noqa: ARG002 + self._note_toolbar_height() self._ready.set() + def _note_toolbar_height(self) -> None: + """Remember how many rows the toolbar took, if the last frame drew it.""" + info = self._toolbar_window.render_info + if info is not None: + self._toolbar_height = info.window_height + def _toolbar_rows(self) -> int: """Return how many rows the toolbar took when it was last drawn.""" - info = self._toolbar_window.render_info - return info.window_height if info is not None else 0 + return self._toolbar_height def start(self) -> None: """Start rendering and protect terminal output.""" diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index c15c53537..ab2531c4c 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -568,6 +568,33 @@ def test_command_toolbar_flushes_writes_waiting_on_cursor_reports() -> None: assert "last words\n" in output.getvalue() +def test_command_toolbar_remembers_the_rows_its_toolbar_took_before_it_is_drawn_again() -> None: + """A command's first output can arrive before the toolbar has been drawn for that command. + + Each run of the application forgets what it drew, and prompt-toolkit leaves the toolbar + out until the terminal has reported where its cursor is. Until then the toolbar counts + for the rows the main prompt drew it with, which are the rows it is about to take again. + """ + app = Cmd(allow_cli_args=False) + output = Terminal() + app.stdout = output + + with create_pipe_input() as pipe: + app.main_session = PromptSession( + input=pipe, + output=CprOutput(output), + bottom_toolbar="one\ntwo", + refresh_interval=0.01, + ) + window = app.main_session.layout.container.children[-1].content + # What the main prompt leaves behind after drawing a toolbar of two rows. + window.render_info = SimpleNamespace(window_height=2) + with app._command_toolbar_context(): + assert window.render_info is None + assert app._command_toolbar._toolbar_rows() == 2 + assert app._command_toolbar._proxy._toolbar_rows() == 2 + + def test_command_toolbar_suspension_waits_for_in_flight_writes(toolbar_app) -> None: app, _, output = toolbar_app writing = threading.Event() From 774721475558451de7af508db7c6f69b5b90f23e Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Sat, 14 Nov 2026 22:39:21 -0500 Subject: [PATCH 33/35] Test Ctrl-Z during an ordinary command, not only in the pager The fix which holds the display thread until a Ctrl-Z stop has taken it was only tested through the built-in pager, which was the convenient way to have a command waiting. The fault is in the command display, whatever the command is doing. Share the test's body and add a command which keeps printing. On Linux with the fix disabled both fail, with the display's cursor reports echoed after fg; with it both pass. --- tests/test_pipeline_job_control.py | 65 ++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/tests/test_pipeline_job_control.py b/tests/test_pipeline_job_control.py index 1ad671faf..64dcfe890 100644 --- a/tests/test_pipeline_job_control.py +++ b/tests/test_pipeline_job_control.py @@ -507,13 +507,18 @@ def wait_until(predicate): process.wait(timeout=5) -def test_pager_resumes_with_raw_terminal_after_suspend(tmp_path) -> None: - """Ctrl-Z in the built-in pager, then fg, must bring the pager back, not echoed cursor reports. +def suspend_and_resume(tmp_path, command_source, command, waiting, finish) -> None: + """Start a command under an interactive bash, stop it with Ctrl-Z, resume it with fg, and let it finish. The display thread sends the stop signal, which the kernel may hand to the main thread. Unless the sending thread stops synchronously too, it restores raw mode before the job has stopped, the shell puts its own modes back on fg, and every cursor-position query afterwards is echoed as ^[[row;colR. + + :param command_source: source of the application's ``do_`` method for the command + :param command: the command to type + :param waiting: returns whether the screen it is given shows the command waiting + :param finish: given ``send``, makes the waiting command finish """ import fcntl import pty @@ -527,8 +532,7 @@ def test_pager_resumes_with_raw_terminal_after_suspend(tmp_path) -> None: application.write_text( "from cmd2 import Cmd\n" "class App(Cmd):\n" - " def do_page(self, statement):\n" - " self.ppaged('\\n'.join(f'line {n:04d}' for n in range(400)))\n" + f"{command_source}" # A refreshing toolbar, as in the examples: its periodic redraws are what turn a cooked terminal into a stream of echoed # cursor reports. "app = App(enable_bottom_toolbar=True, refresh_interval=0.5)\n" @@ -569,8 +573,8 @@ def pump(seconds, predicate=lambda: False): return True return predicate() - def pager_shown(): - return any("q: quit" in line for line in screen.display) + def command_waiting(): + return waiting(screen) try: assert pump(10, lambda: "OUTER> " in transcript) @@ -578,24 +582,23 @@ def pager_shown(): assert pump(10, lambda: "TEST>" in "\n".join(screen.display)), transcript # Let the prompt finish its cursor-position handshake before typing at it. pump(0.5) - send("page\n") - assert pump(10, pager_shown), transcript + send(f"{command}\n") + assert pump(10, command_waiting), transcript start = len(transcript) send("\x1a") assert pump(10, lambda: "OUTER> " in transcript[start:]), transcript start = len(transcript) send("fg\n") - assert pump(10, pager_shown), transcript[start:] + assert pump(10, command_waiting), transcript[start:] # Give a broken resume time to start looping before judging the transcript. pump(1.5) resumed = transcript[start:] echoed = re.findall(r"\^\[\[\d+;\d+R", resumed) assert not echoed, f"cursor position reports were echoed after fg: {echoed[:5]}\n{resumed}" - assert pager_shown(), "\n".join(screen.display) - # A cooked terminal would hold the key back until Enter: the pager must quit on q alone. + assert command_waiting(), "\n".join(screen.display) start = len(transcript) - send("q") - assert pump(10, lambda: "TEST>" in transcript[start:]), f"pager did not quit on q after fg:\n{transcript[start:]}" + finish(send) + assert pump(10, lambda: "TEST>" in transcript[start:]), f"the command did not finish after fg:\n{transcript[start:]}" send("quit\n") pump(5, lambda: os.tcgetpgrp(master) == process.pid) finally: @@ -604,6 +607,42 @@ def pager_shown(): process.wait(timeout=5) +def test_command_resumes_with_raw_terminal_after_suspend(tmp_path) -> None: + """Ctrl-Z while a command runs under the toolbar, then fg, must leave the terminal raw. + + The command keeps printing, and the display asks the terminal where its cursor is as it + puts each line above the toolbar. A terminal left cooked would echo every answer. + """ + release = tmp_path / "release" + + def printing(screen): + return any(line.startswith("waiting") for line in screen.display) + + suspend_and_resume( + tmp_path, + " def do_wait(self, statement):\n" + " import os, time\n" + f" while not os.path.exists({str(release)!r}):\n" + " self.poutput('waiting')\n" + " time.sleep(0.2)\n", + "wait", + printing, + lambda send: release.touch(), + ) + + +def test_pager_resumes_with_raw_terminal_after_suspend(tmp_path) -> None: + """Ctrl-Z in the built-in pager, then fg, must bring the pager back, not echoed cursor reports.""" + suspend_and_resume( + tmp_path, + " def do_page(self, statement):\n self.ppaged('\\n'.join(f'line {n:04d}' for n in range(400)))\n", + "page", + lambda screen: any("q: quit" in line for line in screen.display), + # A cooked terminal would hold the key back until Enter: the pager must quit on q alone. + lambda send: send("q"), + ) + + @pytest.mark.parametrize("parent_delay", [False, True]) def test_pipeline_pager_can_set_terminal_modes_at_startup(tmp_path, parent_delay) -> None: """A pager such as less puts the terminal in raw mode as it starts, before reading its pipe. From ead92b02d8a15685bbd20df30c424336c3fd2e22 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Sat, 14 Nov 2026 22:47:03 -0500 Subject: [PATCH 34/35] Take the built-in pager out, to be proposed on its own The persistent toolbar and the pager which keeps it visible while paging arrived together, but they are two decisions. The pager is about 300 of this branch's 1,050 source lines and a third of its tests, and issue #1743 itself allows the toolbar to disappear "when using a pager". Remove cmd2/pager.py, CommandToolbar.page() and the helpers only it used, the use_builtin_pager attribute, their tests and their documentation. ppaged() goes back to the configured external pager, with the toolbar suspended while it has the terminal, which is the path use_builtin_pager = False already took. The pager returns in a pull request of its own, stacked on this one. --- CHANGELOG.md | 6 - cmd2/cmd2.py | 27 +-- cmd2/command_toolbar.py | 106 +--------- cmd2/pager.py | 229 -------------------- docs/features/initialization.md | 1 - docs/features/os.md | 42 +--- docs/features/prompt.md | 10 +- tests/test_command_toolbar.py | 239 +-------------------- tests/test_pager.py | 323 ----------------------------- tests/test_pipeline_job_control.py | 12 -- 10 files changed, 17 insertions(+), 978 deletions(-) delete mode 100644 cmd2/pager.py delete mode 100644 tests/test_pager.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bc89d83f8..24ce1cba5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,12 +12,6 @@ for input, refreshing at the same `refresh_interval` as the main prompt, instead of the toolbar disappearing for the duration of the nested prompt. `Cmd.select()` is unchanged, since prompt-toolkit's `choice()` offers no way to configure a refresh interval - - Added an embedded pager which `Cmd.ppaged()` uses while the bottom toolbar is running, so the - toolbar stays visible and refreshing instead of the terminal being handed to an external - pager. It supports vertical and horizontal scrolling, incremental search, and chopped lines on - every platform, including Windows where the default external pager (`more`) always wraps. - Output which already fits on the screen is printed directly rather than paged. Set - `self.use_builtin_pager = False` to keep using the external `pager`/`pager_chop` commands. ## 4.4.2 (November 11, 2026) diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index b99ea79e3..8dce5df43 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -661,10 +661,6 @@ def __init__( if callargs: self._startup_commands.extend(callargs) - # The embedded pager shares the main toolbar. Applications can opt back - # into their configured external pager by setting this to False. - self.use_builtin_pager = enable_bottom_toolbar - # Set the pager(s) for use when displaying output using a pager if sys.platform.startswith("win"): self.pager = self.pager_chop = "more" @@ -1956,17 +1952,15 @@ def ppaged( fits on the screen. A pager is not used inside a script (Python or text) or when output is redirected or piped, and in these cases, output is sent to `poutput`. - While the bottom toolbar is running, the built-in pager keeps it visible and refreshing. - Set ``use_builtin_pager=False`` to use the configured external ``pager`` or ``pager_chop`` - command instead; external pagers temporarily hide the toolbar. Where no toolbar is - running, such as outside the command loop, the external pager is used regardless. + The pager takes over the terminal, so the bottom toolbar is hidden while it is open and + comes back when it exits. :param chop: True -> causes lines longer than the screen width to be chopped (truncated) rather than wrapped - truncated text is still accessible by scrolling with the right & left arrow keys - chopping is ideal for displaying wide tabular data as is done in utilities like pgcli False -> causes lines longer than the screen width to wrap to the next line - wrapping is ideal when you want to keep users from having to use horizontal scrolling - WARNING: The default external pager on Windows always wraps; the built-in pager supports chopping. + WARNING: On Windows, the text always wraps regardless of what the chop argument is set to :param soft_wrap: Enable soft wrap mode. If True, lines of text will not be word-wrapped or cropped to fit the terminal width. Defaults to True. @@ -2009,17 +2003,8 @@ def ppaged( soft_wrap=soft_wrap, **(rich_print_kwargs if rich_print_kwargs is not None else {}), ) - output = capture.get() - - # Page inside the toolbar's display only when the command loop is already - # running one. Starting one here would seize the terminal for commands run - # outside that loop, which the toolbar is documented not to do. - if self.use_builtin_pager and self._command_toolbar is not None and self._command_toolbar.is_active: - self._command_toolbar.page(output, chop=chop) - return - # As for a pipe: the pager gets UTF-8, and on Windows the console is UTF-8 while it runs. - output_bytes = output.encode("utf-8", "replace") + output_bytes = capture.get().encode("utf-8", "replace") # Prevent KeyboardInterrupts while in the pager. The pager application will # still receive the SIGINT since it is in the same process group as us. @@ -2131,8 +2116,8 @@ def get_bottom_toolbar(self) -> AnyFormattedText: During command execution this callback runs in a background UI thread. Protect shared state with a lock when necessary, but never hold that lock in a command while it prompts for input, pages output, runs a shell command, or otherwise pauses the toolbar: pausing - waits for this callback to return. The built-in pager shares this toolbar. It is suspended - while another prompt, external pager, or interactive shell owns the terminal. + waits for this callback to return. The toolbar is suspended while another prompt, a + pager, or an interactive shell owns the terminal. If this callback raises an exception during a command, the error is reported once and the command carries on without the toolbar. diff --git a/cmd2/command_toolbar.py b/cmd2/command_toolbar.py index 0d59e3e9e..dc709f1e3 100644 --- a/cmd2/command_toolbar.py +++ b/cmd2/command_toolbar.py @@ -12,12 +12,9 @@ import threading import time from collections.abc import Callable, Generator -from concurrent.futures import Future -from concurrent.futures import TimeoutError as FutureTimeoutError from typing import TYPE_CHECKING, Any, TextIO, TypeVar, cast from prompt_toolkit.application import Application, create_app_session, run_in_terminal -from prompt_toolkit.enums import EditingMode from prompt_toolkit.filters import Condition, to_filter from prompt_toolkit.input.typeahead import get_typeahead, store_typeahead from prompt_toolkit.key_binding import KeyBindings @@ -27,8 +24,6 @@ from prompt_toolkit.patch_stdout import StdoutProxy from prompt_toolkit.utils import get_cwidth, suspend_to_background_supported -from .pager import Pager, output_fits - if TYPE_CHECKING: from .cmd2 import Cmd @@ -37,7 +32,6 @@ _ESCAPE_RE = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|\][^\x07\x1b]*(?:\x07|\x1b\\)|[@-Z\\-_])") _F = TypeVar("_F", bound=Callable[..., Any]) -_R = TypeVar("_R") def suspend_process_group(suspend_group: bool = True) -> None: @@ -347,7 +341,6 @@ def suspend(event: KeyPressEvent) -> None: event.app.suspend_to_background() self._bindings = bindings - self._suspend_binding = suspend @staticmethod def _suspend_to_background(suspend_group: bool = True) -> None: @@ -405,7 +398,7 @@ def _resume(self) -> None: # has not seen with other bindings. It caches the bindings it merges for each focused # window, application bindings included, and does not notice this attribute being # reassigned, so reusing a window with different bindings would keep the old ones. - # This layout, the main prompt's, and each pager's all have windows of their own. + # This layout and the main prompt's have windows of their own. for name, value in (("layout", self._layout), ("key_bindings", self._bindings), ("erase_when_done", True)): stack.callback(setattr, self.app, name, getattr(self.app, name)) setattr(self.app, name, value) @@ -560,103 +553,6 @@ def is_active(self) -> bool: """Whether the display currently owns the terminal.""" return self._proxy is not None and self.app.is_running - def _call_in_ui(self, func: Callable[[], _R]) -> _R: - """Change UI state on its event loop, propagating failures to the command.""" - result: Future[_R] = Future() - - def call() -> None: - try: - value = func() - except BaseException as exc: # noqa: BLE001 - result.set_exception(exc) - else: - result.set_result(value) - - if self.app.loop is None: - raise RuntimeError("Toolbar is not running") - self.app.loop.call_soon_threadsafe(call) - while True: - try: - value = result.result(timeout=0.1) - except FutureTimeoutError: - if result.done(): - # The callback finished while this poll was expiring, or raised a - # TimeoutError of its own -- indistinguishable here, because - # concurrent.futures.TimeoutError is TimeoutError on Python 3.11+. - # Ask the future for its outcome rather than re-raising this poll's - # timeout, which would report a failure for a call that succeeded. - return result.result() - self._check_running() - else: - return value - - def _check_running(self) -> None: - if self._thread is None or not self._thread.is_alive(): - if self._error is not None: - raise self._error - raise EOFError - - def page(self, text: str, *, chop: bool) -> None: - """Show a pager above the same toolbar without starting another input reader.""" - size = self.app.output.get_size() - # Measuring the toolbar can invoke its callback; keep that work on the - # UI thread along with rendering and layout changes. - toolbar_height = self._call_in_ui(lambda: self.toolbar.preferred_height(size.columns, size.rows).preferred) - if output_fits(text, size.columns, max(0, size.rows - toolbar_height), chop=chop): - self.cmd.stdout.write(text) - self.cmd.stdout.flush() - return - - pager = Pager(text, chop=chop) - pager.bindings.add( - "c-z", - filter=Condition(lambda: suspend_to_background_supported() and to_filter(self.cmd.main_session.enable_suspend)()), - )(self._suspend_binding) - layout = Layout(HSplit([pager.container, self.toolbar]), focused_element=pager.text) - previous = (self.app.layout, self.app.key_bindings, self.app.editing_mode, self.app.full_screen) - entered = False - - def enter() -> None: - nonlocal entered - entered = True - self.app.renderer.erase() - self.app.layout = layout - self.app.key_bindings = pager.bindings - self.app.editing_mode = EditingMode.EMACS - self.app.full_screen = self.app.renderer.full_screen = True - self.app.invalidate() - - def leave() -> None: - nonlocal entered - if not entered: - return - entered = False - self.app.renderer.erase() - self.app.layout, self.app.key_bindings, self.app.editing_mode, self.app.full_screen = previous - self.app.renderer.full_screen = self.app.full_screen - self.app.renderer.request_absolute_cursor_position() - self.app.invalidate() - - def close() -> None: - # Switch bindings before the next key is processed, preserving - # typeahead sent in the same terminal read as the pager's quit key. - leave() - pager.closed.set() - - pager.on_close = close - - try: - self._call_in_ui(enter) - while not pager.closed.wait(0.1): - self._check_running() - finally: - if self._thread is not None and self._thread.is_alive(): - self._call_in_ui(leave) - else: - # The application's shutdown already reset the renderer. - self.app.layout, self.app.key_bindings, self.app.editing_mode, self.app.full_screen = previous - self.app.renderer.full_screen = self.app.full_screen - @contextlib.contextmanager def suspend(self) -> Generator[None]: """Temporarily restore ordinary terminal access, including nested suspensions.""" diff --git a/cmd2/pager.py b/cmd2/pager.py deleted file mode 100644 index 438dd4e48..000000000 --- a/cmd2/pager.py +++ /dev/null @@ -1,229 +0,0 @@ -"""A pager view hosted by the main prompt-toolkit application.""" - -import re -import threading -from collections.abc import Callable -from functools import partial - -from prompt_toolkit.document import Document -from prompt_toolkit.filters import has_focus, is_searching, to_filter -from prompt_toolkit.formatted_text import ANSI, fragment_list_to_text, to_formatted_text -from prompt_toolkit.formatted_text.base import StyleAndTextTuples -from prompt_toolkit.formatted_text.utils import split_lines -from prompt_toolkit.key_binding import KeyBindings -from prompt_toolkit.key_binding.bindings import search -from prompt_toolkit.key_binding.key_processor import KeyPressEvent -from prompt_toolkit.layout import HSplit, Window -from prompt_toolkit.layout.containers import ConditionalContainer -from prompt_toolkit.layout.controls import FormattedTextControl, UIContent -from prompt_toolkit.lexers import Lexer -from prompt_toolkit.search import SearchDirection, start_search -from prompt_toolkit.utils import get_cwidth -from prompt_toolkit.widgets import SearchToolbar, TextArea - -_OSC_RE = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)") - - -def _fragments(text: str) -> StyleAndTextTuples: - """Parse captured output, dropping the trailing newline that ends its last line.""" - # prompt-toolkit's ANSI parser does not handle OSC sequences, including Rich - # hyperlinks. Remove their metadata while retaining visible text and SGR styles. - text = _OSC_RE.sub("", text) - return to_formatted_text(ANSI(text.removesuffix("\n"))) - - -def output_fits(text: str, columns: int, rows: int, *, chop: bool) -> bool: - """Check rendered line heights, including wrapping and wide Unicode characters. - - Measuring the text itself keeps output that needs no scrolling from paying for a - Pager's widgets and key bindings, which would only be built to be thrown away. - """ - lines = fragment_list_to_text(_fragments(text)).split("\n") - if chop: - # Even one wide line needs a pager so its hidden columns remain - # accessible through horizontal scrolling. - return len(lines) <= rows and all(get_cwidth(line) <= columns for line in lines) - # Reuse prompt-toolkit's own wrapping arithmetic so this matches what a Pager - # would render, without building a control to ask on the UI thread. - content = UIContent(get_line=lambda number: [("", lines[number])], line_count=len(lines)) - height = 0 - for line in range(content.line_count): - height += content.get_height_for_line(line, columns, None) - if height > rows: - return False - return True - - -class _AnsiLexer(Lexer): - """Preserve captured Rich styles while searching and scrolling plain text.""" - - def __init__(self, fragments: StyleAndTextTuples) -> None: - self.lines = list(split_lines(fragments)) - - def lex_document(self, document: Document) -> Callable[[int], StyleAndTextTuples]: # noqa: ARG002 - def get_line(number: int) -> StyleAndTextTuples: - return self.lines[number] if number < len(self.lines) else [] - - return get_line - - -class Pager: - """Scrollable, searchable output; the host supplies the persistent toolbar.""" - - def __init__(self, text: str, *, chop: bool) -> None: - """Build an independent view without creating an application or input reader.""" - self.closed = threading.Event() - self.on_close: Callable[[], None] = self.closed.set - self.chop = chop - fragments = _fragments(text) - self.search = SearchToolbar() - self.text = TextArea( - text=fragment_list_to_text(fragments), - lexer=_AnsiLexer(fragments), - read_only=True, - wrap_lines=not chop, - search_field=self.search, - ) - self.text.window.always_hide_cursor = to_filter(True) - self.container = HSplit( - [ - self.text, - self.search, - ConditionalContainer( - Window( - FormattedTextControl(" Space/PgDn: next b/PgUp: back /: search n: next match q: quit"), - height=1, - style="class:bottom-toolbar", - ), - filter=~is_searching, - ), - ] - ) - self.bindings = bindings = KeyBindings() - focused = has_focus(self.text) - - @bindings.add("", filter=focused) - def ignore(event: KeyPressEvent) -> None: - # Pager keystrokes must not become commands at the next prompt. - pass - - @bindings.add("q", filter=focused) - @bindings.add("escape", filter=focused, eager=True) - @bindings.add("c-c", filter=focused) - def close(event: KeyPressEvent) -> None: # noqa: ARG001 - self.on_close() - - for keys, pages in ( - ((" ", "pagedown", "f", "c-f"), 1.0), - (("b", "pageup", "c-b"), -1.0), - (("d", "c-d"), 0.5), - (("u", "c-u"), -0.5), - ): - for key in keys: - bindings.add(key, filter=focused)(partial(self._scroll_page, pages=pages)) - - @bindings.add("j", filter=focused) - @bindings.add("down", filter=focused) - @bindings.add("enter", filter=focused) - def down(event: KeyPressEvent) -> None: - self._scroll(event, 1) - - @bindings.add("k", filter=focused) - @bindings.add("up", filter=focused) - def up(event: KeyPressEvent) -> None: - self._scroll(event, -1) - - @bindings.add("right", filter=focused) - @bindings.add("l", filter=focused) - def right(event: KeyPressEvent) -> None: - self._scroll_horizontal(event, 1) - - @bindings.add("left", filter=focused) - @bindings.add("h", filter=focused) - def left(event: KeyPressEvent) -> None: - self._scroll_horizontal(event, -1) - - @bindings.add("g", filter=focused) - @bindings.add("home", filter=focused) - def first(event: KeyPressEvent) -> None: - event.current_buffer.cursor_position = 0 - - @bindings.add("G", filter=focused) - @bindings.add("end", filter=focused) - def last(event: KeyPressEvent) -> None: - event.current_buffer.cursor_position = len(event.current_buffer.text) - - @bindings.add("/", filter=focused) - def find(event: KeyPressEvent) -> None: # noqa: ARG001 - start_search(direction=SearchDirection.FORWARD) - - @bindings.add("?", filter=focused) - def find_backwards(event: KeyPressEvent) -> None: # noqa: ARG001 - start_search(direction=SearchDirection.BACKWARD) - - @bindings.add("n", filter=focused) - def next_match(event: KeyPressEvent) -> None: - event.current_buffer.apply_search(event.app.current_search_state, include_current_position=False) - - @bindings.add("N", filter=focused) - def previous_match(event: KeyPressEvent) -> None: - event.current_buffer.apply_search(~event.app.current_search_state, include_current_position=False) - - # Explicit search bindings also work when the main prompt uses Vi mode. - bindings.add("enter", filter=is_searching)(search.accept_search) - bindings.add("escape", filter=is_searching, eager=True)(search.abort_search) - bindings.add("c-c", filter=is_searching)(search.abort_search) - - @staticmethod - def _column_at_width(line: str, width: int) -> int: - used = 0 - for column, char in enumerate(line): - if used >= width: - return column - used += get_cwidth(char) - return len(line) - - def _scroll_page(self, event: KeyPressEvent, *, pages: float) -> None: - info = self.text.window.render_info - if info is not None: - amount = max(1, int(max(1, info.window_height - 1) * abs(pages))) - self._scroll(event, amount if pages > 0 else -amount) - - def _scroll(self, event: KeyPressEvent, rows: int) -> None: - """Move by display rows, including within lines taller than the viewport.""" - info = self.text.window.render_info - if info is None or info.window_width == 0: - return - document = event.current_buffer.document - line = document.cursor_position_row - wrapped_row = 0 if self.chop else get_cwidth(document.current_line_before_cursor) // info.window_width - target = wrapped_row + rows - - def height(number: int) -> int: - return 1 if self.chop else info.get_height_for_line(number) - - while target < 0 and line > 0: - line -= 1 - target += height(line) - while target >= height(line) and line < document.line_count - 1: - target -= height(line) - line += 1 - target = max(0, min(target, height(line) - 1)) - # Chopped lines never wrap, so leave the cursor at the column the reader - # scrolled to. Moving it to the start of the line would drag the view back - # with it, since the window scrolls horizontally to keep the cursor visible. - width = self.text.window.horizontal_scroll if self.chop else target * info.window_width - column = self._column_at_width(document.lines[line], width) - event.current_buffer.cursor_position = document.translate_row_col_to_index(line, column) - self.text.window.vertical_scroll = line - self.text.window.vertical_scroll_2 = target if height(line) > info.window_height else 0 - - def _scroll_horizontal(self, event: KeyPressEvent, direction: int) -> None: - info = self.text.window.render_info - if info is None or not self.chop: - return - document = event.current_buffer.document - target = max(0, self.text.window.horizontal_scroll + direction * max(1, info.window_width // 2)) - column = self._column_at_width(document.current_line, target) - event.current_buffer.cursor_position = document.translate_row_col_to_index(document.cursor_position_row, column) - self.text.window.horizontal_scroll = get_cwidth(document.current_line[:column]) diff --git a/docs/features/initialization.md b/docs/features/initialization.md index 88e6d79d7..fc084ff3d 100644 --- a/docs/features/initialization.md +++ b/docs/features/initialization.md @@ -52,7 +52,6 @@ Here are instance attributes of `cmd2.Cmd` which developers might wish to overri - **max_completion_table_items**: The maximum number of completion results allowed for a completion table to appear (Default: 50) - **pager**: sets the pager command used by the `Cmd.ppaged()` method for displaying wrapped output using a pager - **pager_chop**: sets the pager command used by the `Cmd.ppaged()` method for displaying chopped/truncated output using a pager -- **use_builtin_pager**: defaults to `enable_bottom_toolbar`. While the command toolbar is running, `Cmd.ppaged()` uses an embedded pager that keeps that toolbar visible. Set to `False` to always use the external `pager`/`pager_chop` commands. This is an opt-*out* only: setting it to `True` does nothing unless `enable_bottom_toolbar` is also set, because the embedded pager shares the toolbar's display. - **py_bridge_name**: name by which embedded Python environments and scripts refer to the `cmd2` application by in order to call commands (Default: `app`) - **py_locals**: dictionary that defines specific variables/functions available in Python shells and scripts (provides more fine-grained control than making everything available with **self_in_py**) - **quiet**: if `True`, then completely suppress nonessential output (Default: `False`) diff --git a/docs/features/os.md b/docs/features/os.md index 0706c7a69..f4d399598 100644 --- a/docs/features/os.md +++ b/docs/features/os.md @@ -46,44 +46,8 @@ Output of any command can be displayed one page at a time using the [cmd2.Cmd.pp pager is only used when the terminal is interactive. Inside a script, or when the command's output is redirected or piped, `ppaged()` falls back to `poutput()`. -### Embedded pager - -While the [bottom toolbar](./prompt.md#bottom-toolbar) is running, `ppaged()` displays its output in -an embedded pager rather than handing the terminal to an external program. The toolbar stays visible -and keeps refreshing beneath the paged output, and `cmd2` never gives up control of the terminal. - -Output which already fits in the space above the toolbar is printed directly, so there is no pager -to dismiss. - -| Keys | Action | -| ---------------------------------- | ------------------------------------------------- | -| `Space`, `PageDown`, `f`, `Ctrl-F` | Forward one page | -| `b`, `PageUp`, `Ctrl-B` | Back one page | -| `d` / `u`, `Ctrl-D` / `Ctrl-U` | Forward / back half a page | -| `j` / `k`, `Down` / `Up`, `Enter` | Down / up one line | -| `h` / `l`, `Left` / `Right` | Scroll left / right (chopped output only) | -| `g` / `G`, `Home` / `End` | Jump to the beginning / end | -| `/` / `?` | Search forward / backward | -| `n` / `N` | Jump to the next / previous match | -| `q`, `Escape`, `Ctrl-C` | Close the pager and return to the running command | -| `Ctrl-Z` | Suspend to the background, where supported | - -Passing `chop=True` truncates lines longer than the screen width instead of wrapping them, and the -hidden columns remain reachable with the left and right arrow keys. Unlike the default external -pager on Windows (`more`), the embedded pager supports chopping on every platform. - -Rich styles in the paged output are preserved: colors are rendered while scrolling and searching, -and the visible text of a hyperlink is shown without its escape sequence metadata. - -### External pagers - -Set `self.use_builtin_pager = False` to always use the external commands configured in `self.pager` -and `self.pager_chop`. This is the opt-out for applications which need `less` features the embedded -pager does not provide. External pagers temporarily hide the toolbar and restore it on exit. - -The setting only ever turns the embedded pager off. Because the embedded pager shares the toolbar's -display, it is unavailable anywhere a toolbar is not running, such as a command invoked outside the -command loop, and `ppaged()` uses the external pager there regardless of this setting. +While the [bottom toolbar](./prompt.md#bottom-toolbar) is running, a pager temporarily hides it, +because the pager takes over the terminal. The toolbar comes back when the pager exits. Alternatively, a terminal pager can be invoked directly using the ability to run shell commands with the `!` shortcut like so: @@ -92,7 +56,7 @@ the `!` shortcut like so: !!! warning - Once you are in an external terminal pager, that program temporarily has control of your terminal, + Once you are in a terminal pager, that program temporarily has control of your terminal, **NOT** `cmd2`. Typically you can use either the arrow keys or ``/`` keys to scroll around or type `q` to quit the pager and return control to your `cmd2` application. diff --git a/docs/features/prompt.md b/docs/features/prompt.md index 318e37138..698d1ff53 100644 --- a/docs/features/prompt.md +++ b/docs/features/prompt.md @@ -153,14 +153,8 @@ once and the command carries on without the toolbar. ### Commands That Take Over the Terminal -While the toolbar is running, `ppaged()` uses an embedded pager which keeps the same toolbar visible -and refreshing instead of handing the terminal to an external pager. Short output is printed -directly above the toolbar, and longer output becomes a scrollable, searchable view. Set -`self.use_builtin_pager = False` to opt out and use your configured external `pager`/`pager_chop` -commands. See [Embedded pager](./os.md#embedded-pager) for the key bindings and full details. - -cmd2 temporarily hides the toolbar for external pagers, Python environments, and shell commands. It -also hides it while a command's output is piped to another process, since that process may be +cmd2 temporarily hides the toolbar for pagers, Python environments, and shell commands. It also +hides it while a command's output is piped to another process, since that process may be interactive, as `less` and `fzf` are. It restores the toolbar when those operations finish. [cmd2.Cmd.read_input][] and [cmd2.Cmd.read_secret][] keep the toolbar visible, and it continues to diff --git a/tests/test_command_toolbar.py b/tests/test_command_toolbar.py index ab2531c4c..18b1ffe9b 100644 --- a/tests/test_command_toolbar.py +++ b/tests/test_command_toolbar.py @@ -3,8 +3,6 @@ import sys import threading import time -from concurrent.futures import Future, ThreadPoolExecutor -from concurrent.futures import TimeoutError as FutureTimeoutError from types import SimpleNamespace from unittest import mock @@ -110,10 +108,8 @@ def __getattr__(self, name): return getattr(self.file, name) -@pytest.mark.parametrize("builtin_pager", [False, True]) -def test_command_toolbar_pipe_process_inherits_terminal(toolbar_app, tmp_path, builtin_pager) -> None: +def test_command_toolbar_pipe_process_inherits_terminal(toolbar_app, tmp_path) -> None: app, _, _ = toolbar_app - app.use_builtin_pager = builtin_pager destination = tmp_path / "terminal.txt" running = [] readers = [] @@ -778,6 +774,7 @@ def failing_stop() -> None: def test_command_toolbar_exit_after_result_is_set(toolbar_app) -> None: app, _, _ = toolbar_app + exited_twice = threading.Event() with app._command_toolbar_context(): toolbar = app._command_toolbar @@ -786,110 +783,13 @@ def already_exiting(): # The result is set before run_async() has finished its cleanup. assert toolbar.app.is_running toolbar._exit() + exited_twice.set() - toolbar._call_in_ui(already_exiting) + toolbar.app.loop.call_soon_threadsafe(already_exiting) + assert exited_twice.wait(2) assert app.main_session.app.layout is app.main_session.layout -def test_command_toolbar_ui_call_propagates_failures(toolbar_app) -> None: - app, _, _ = toolbar_app - - def fail(exception: BaseException) -> None: - raise exception - - with app._command_toolbar_context(): - toolbar = app._command_toolbar - - # A UI callback runs on the display's loop, so its failure has to be carried - # back to the command thread rather than reaching the loop's error handler. - with pytest.raises(ValueError, match="broken ui call"): - toolbar._call_in_ui(lambda: fail(ValueError("broken ui call"))) - - # A TimeoutError raised by the callback is the same class the pending future - # reports itself with, and must not be mistaken for one. - with pytest.raises(TimeoutError, match="slow ui call"): - toolbar._call_in_ui(lambda: fail(TimeoutError("slow ui call"))) - - # A callback that outlives the poll interval keeps waiting instead of giving up. - assert toolbar._call_in_ui(lambda: time.sleep(0.2) or "finished") == "finished" - - -def test_command_toolbar_ui_call_returns_a_result_that_lands_during_the_poll(toolbar_app, monkeypatch) -> None: - """A callback finishing while the poll expires must return its value, not a timeout. - - `concurrent.futures.TimeoutError` is `TimeoutError` on Python 3.11+, so the poll - expiring and the callback raising a timeout of its own are indistinguishable by type. - Re-raising the caught exception once the future is done therefore reports a timeout for - a call that actually succeeded. - """ - app, _, _ = toolbar_app - - class RacyFuture(Future): - """Completes, and only then reports the poll as having expired.""" - - def __init__(self) -> None: - super().__init__() - self._polled = False - - def result(self, timeout=None): # type: ignore[no-untyped-def] - if timeout is not None and not self._polled: - self._polled = True - super().result(timeout=5) # let the callback finish first - raise FutureTimeoutError # then act as though the poll had expired - return super().result(timeout) - - monkeypatch.setattr(command_toolbar, "Future", RacyFuture) - with app._command_toolbar_context(): - toolbar = app._command_toolbar - assert toolbar._call_in_ui(lambda: "finished") == "finished" - - -def test_command_toolbar_ui_call_after_display_stopped(toolbar_app) -> None: - app, pipe, _ = toolbar_app - with app._command_toolbar_context(): - toolbar = app._command_toolbar - pipe.close() - toolbar._thread.join(timeout=2) - assert not toolbar._thread.is_alive() - - # There is no loop left to run UI work on, so asking must fail rather than - # queue a callback onto a closed loop. - assert toolbar.app.loop is None - with pytest.raises(RuntimeError, match="Toolbar is not running"): - toolbar._call_in_ui(lambda: None) - - -def test_command_toolbar_ui_call_reports_display_failure(toolbar_app, capsys) -> None: - app, _, _ = toolbar_app - with app._command_toolbar_context(): - toolbar = app._command_toolbar - loop = toolbar.app.loop - schedule = loop.call_soon_threadsafe - failed = threading.Event() - - def die(*args, **kwargs): - # The display dies instead of running the queued callback, so the future - # the command is waiting on never resolves. Only drop that one request, - # made here on the command thread. asyncio uses call_soon_threadsafe from - # its own threads, and on Windows the default executor's join is reported - # through it while asyncio.run() shuts the loop down. Swallowing that - # report strands the toolbar thread for 300 seconds, or forever before - # Python 3.12, where shutdown_default_executor() has no timeout. - if not failed.is_set() and threading.current_thread() is threading.main_thread(): - failed.set() - schedule(lambda: toolbar.app.exit(exception=ValueError("broken display"))) - return None - return schedule(*args, **kwargs) - - with ( - mock.patch.object(loop, "call_soon_threadsafe", side_effect=die), - pytest.raises(ValueError, match="broken display"), - ): - toolbar._call_in_ui(lambda: None) - - assert "broken display" in capsys.readouterr().err - - def test_command_toolbar_failure_after_startup_is_reported(toolbar_app, capsys) -> None: app, _, output = toolbar_app @@ -1160,107 +1060,8 @@ def test_command_toolbar_reuses_prompt_application(toolbar_app) -> None: assert app._read_raw_input("Next: ", session, pre_run=lambda: pipe.send_text("answer\n")) == "answer" -@pytest.mark.parametrize("quit_key", ["q", "\x03"]) -def test_builtin_pager_keeps_toolbar_live(toolbar_app, monkeypatch, quit_key) -> None: - app, pipe, _ = toolbar_app - app.use_builtin_pager = True - monkeypatch.setattr(app, "stdin", Terminal()) - monkeypatch.setenv("TERM", "xterm") - entered, refreshed, moved, found = (threading.Event() for _ in range(4)) - state = ["BEFORE"] - - def toolbar_text(): - assert threading.current_thread() is not threading.main_thread() - if app.main_session.app.full_screen and state[0] == "AFTER": - refreshed.set() - return state[0] - - app.main_session.bottom_toolbar = toolbar_text - prompt_layout = app.main_session.app.layout - - def observe(ui): - if not ui.full_screen: - return - assert ui.layout.container.children[-1] is app._command_toolbar.toolbar - entered.set() - row = ui.layout.current_buffer.document.cursor_position_row - if row > 0: - moved.set() - if row == 80: - found.set() - - def interact(): - try: - assert entered.wait(2) - state[0] = "AFTER" - assert refreshed.wait(2) - pipe.send_text(" ") - assert moved.wait(2) - pipe.send_text("/row 080\n") - assert found.wait(2) - finally: - pipe.send_text(quit_key) - - app.main_session.app.after_render += observe - with mock.patch("subprocess.Popen") as external, ThreadPoolExecutor() as executor: - interaction = executor.submit(interact) - with app._command_toolbar_context(): - thread = app._command_toolbar._thread - app.ppaged("\n".join(f"row {i:03d}" for i in range(100))) - assert app._command_toolbar._thread is thread - assert app._command_toolbar.is_active - assert not app.main_session.app.full_screen - assert not app.main_session.app.renderer.full_screen - interaction.result(timeout=2) - external.assert_not_called() - assert app.main_session.app.layout is prompt_layout - assert refreshed.is_set() - assert get_typeahead(pipe) == [] - - -def test_builtin_pager_short_output(toolbar_app, monkeypatch) -> None: - app, _, output = toolbar_app - app.use_builtin_pager = True - monkeypatch.setattr(app, "stdin", Terminal()) - monkeypatch.setenv("TERM", "xterm") - with mock.patch("subprocess.Popen") as external, app._command_toolbar_context(): - app.ppaged("short output") - assert app._command_toolbar.is_active - external.assert_not_called() - assert "short output\n" in output.getvalue() - - -def test_builtin_pager_short_output_builds_no_pager(toolbar_app, monkeypatch) -> None: - app, _, output = toolbar_app - app.use_builtin_pager = True - monkeypatch.setattr(app, "stdin", Terminal()) - monkeypatch.setenv("TERM", "xterm") - # Output that fits is written directly, so none of the pager's widgets are needed. - with mock.patch("cmd2.command_toolbar.Pager") as pager, app._command_toolbar_context(): - app.ppaged("short output") - pager.assert_not_called() - assert "short output\n" in output.getvalue() - - -def test_builtin_pager_needs_an_already_running_toolbar(toolbar_app, monkeypatch) -> None: - app, _, _ = toolbar_app - app.use_builtin_pager = True - monkeypatch.setattr(app, "stdin", Terminal()) - monkeypatch.setenv("TERM", "xterm") - # Outside the command loop there is no toolbar to page inside. Starting one here - # would wrap the terminal streams and enter raw mode where the docs promise not to. - with ( - mock.patch.object(command_toolbar.CommandToolbar, "start") as start, - mock.patch("subprocess.Popen") as external, - ): - app.ppaged("row\n" * 200) - start.assert_not_called() - external.assert_called_once() - - def test_external_pager_suspends_shared_application(toolbar_app, monkeypatch) -> None: app, _, _ = toolbar_app - app.use_builtin_pager = False monkeypatch.setattr(app, "stdin", Terminal()) monkeypatch.setenv("TERM", "xterm") @@ -1272,33 +1073,3 @@ def external(*args, **kwargs): with mock.patch("subprocess.Popen", side_effect=external), app._command_toolbar_context(): app.ppaged("external pager") assert app._command_toolbar.is_active - - -def test_builtin_pager_eof_restores_prompt(toolbar_app) -> None: - app, pipe, output = toolbar_app - layout = app.main_session.app.layout - - def close_input(ui): - if ui.full_screen: - pipe.close() - - app.main_session.app.after_render += close_input - with pytest.raises(EOFError), app._command_toolbar_context(): - app._command_toolbar.page("line\n" * 100, chop=False) - assert app.main_session.app.layout is layout - assert not app.main_session.app.full_screen - assert not app.main_session.app.renderer.full_screen - assert app.stdout is output - - -def test_builtin_pager_does_not_capture_redirected_output(toolbar_app, monkeypatch, tmp_path) -> None: - app, _, output = toolbar_app - app.use_builtin_pager = True - monkeypatch.setattr(app, "stdin", Terminal()) - monkeypatch.setenv("TERM", "xterm") - target = tmp_path / "help.txt" - with mock.patch("cmd2.command_toolbar.Pager") as pager, app._command_toolbar_context(): - app.onecmd_plus_hooks(f'help > "{target}"') - pager.assert_not_called() - assert "Cmd2 Commands" in target.read_text(encoding="utf-8") - assert "Cmd2 Commands" not in output.getvalue() diff --git a/tests/test_pager.py b/tests/test_pager.py deleted file mode 100644 index b53aff4de..000000000 --- a/tests/test_pager.py +++ /dev/null @@ -1,323 +0,0 @@ -"""Tests for the built-in pager view.""" - -import threading -import time -from concurrent.futures import ThreadPoolExecutor -from types import SimpleNamespace -from unittest import mock - -import pytest -from prompt_toolkit.data_structures import Size -from prompt_toolkit.input.typeahead import get_typeahead -from rich.console import Console - -from cmd2.pager import Pager, output_fits - - -@pytest.mark.parametrize("chop", [False, True]) -def test_pager_styles_and_wrapping(chop) -> None: - pager = Pager("\x1b[31m" + "η•Œ" * 40 + "\x1b[0m\n", chop=chop) - assert pager.text.text == "η•Œ" * 40 - assert pager.text.read_only - lexer = pager.text.lexer.lex_document(pager.text.document) - assert "ansired" in lexer(0)[0][0] - - -@pytest.mark.parametrize("chop", [False, True]) -@pytest.mark.parametrize("terminator", ["\x1b\\", "\x07"], ids=["ST", "BEL"]) -def test_pager_rich_hyperlinks(chop, terminator) -> None: - # legacy_windows=False keeps rich from suppressing OSC 8 hyperlinks when the - # tests run against a legacy Windows console. - console = Console(force_terminal=True, color_system="standard", no_color=False, legacy_windows=False) - with console.capture() as capture: - console.print("[link=https://example.com][red]click here[/red][/link] after") - captured = capture.get() - assert "\x1b]8;" in captured - captured = captured.replace("\x1b\\", terminator) - - pager = Pager(captured, chop=chop) - assert pager.text.text == "click here after" - lexer = pager.text.lexer.lex_document(pager.text.document) - assert "ansired" in lexer(0)[0][0] - assert output_fits(captured, len("click here after"), 1, chop=chop) - assert not output_fits(captured, len("click here after") - 1, 1, chop=chop) - - -@pytest.mark.parametrize("chop", [False, True]) -def test_output_fits_measures_styled_and_wide_text(chop) -> None: - # Forty double-width characters occupy eighty columns, and styling them adds - # escape sequences that must not count towards the measurement. - text = "\x1b[31m" + "η•Œ" * 40 + "\x1b[0m\n" - assert not output_fits(text, 20, 1, chop=chop) - # Wrapping the line onto four rows fits; chopping keeps it one wide row that does not. - assert output_fits(text, 20, 5, chop=chop) is (not chop) - assert output_fits(text, 100, 1, chop=chop) - - -@pytest.mark.parametrize("chop", [False, True]) -def test_pager_long_line_navigation_resize_and_typeahead(toolbar_app, chop) -> None: - app, pipe, _ = toolbar_app - app.main_session.bottom_toolbar = "STATUS ONE\nSTATUS TWO" - entered, scrolled, resized = (threading.Event() for _ in range(3)) - - def observe(ui): - if not ui.full_screen: - return - # Check the rendered frame, not the stream of incremental terminal - # writes, to verify both toolbar rows survive navigation and resizing. - screen = ui.renderer._last_screen - # Use the size this frame was drawn at. The terminal may have been resized since, - # and an error raised here stops the display. - size = ui.renderer._last_size - bottom = "".join(screen.data_buffer[size.rows - 1][x].char for x in range(size.columns)) - assert bottom.startswith("STATUS TWO") - entered.set() - if ui.current_buffer.cursor_position > 0: - scrolled.set() - if size.columns == 60: - resized.set() - - def interact(): - try: - assert entered.wait(2) - pipe.send_text("\x1b[C" if chop else " ") - assert scrolled.wait(2) - app.main_session.output.size = Size(rows=20, columns=60) - app.main_session.app.invalidate() - assert resized.wait(2) - finally: - pipe.send_text("qnext\n") - - app.main_session.app.after_render += observe - with ThreadPoolExecutor() as executor: - interaction = executor.submit(interact) - with app._command_toolbar_context(): - app._command_toolbar.page("η•Œ" * 4000, chop=chop) - interaction.result(timeout=2) - assert app._read_raw_input("Next: ", app.main_session) == "next" - - -class PagerKeys: - """Send keys to the built-in pager and wait for the frame that reflects them.""" - - def __init__(self, app, pipe, pager) -> None: - self.pipe = pipe - self.pager = pager - self.presses = 0 - self.states = [] - self.updated = threading.Condition() - ui = app.main_session.app - ui.key_processor.after_key_press += self._count - ui.after_render += self._record - - def _count(self, _) -> None: - with self.updated: - self.presses += 1 - - def _record(self, _) -> None: - # Read the pager's own buffer, not the focused one, which is the search field - # while a search is being typed. Read it after rendering, because - # prompt-toolkit settles the window's scroll offsets while it draws. - with self.updated: - self.states.append( - ( - self.presses, - self.pager.text.buffer.document.cursor_position_row, - self.pager.text.window.horizontal_scroll, - ) - ) - self.updated.notify_all() - - def press(self, keys, row, column=0) -> None: - """Send keys and wait for a drawn frame that shows the expected position.""" - with self.updated: - handled = self.presses - self.pipe.send_text(keys) - deadline = time.monotonic() + 5 - index = 0 - with self.updated: - while True: - while index < len(self.states): - presses, *position = self.states[index] - index += 1 - if presses > handled and position == [row, column]: - return - remaining = deadline - time.monotonic() - notified = remaining > 0 and self.updated.wait(remaining) - assert notified, f"pager ignored {keys!r}: wanted {(row, column)}, saw {self.states[-3:]}" - - -def drive_pager(app, pipe, text, *, chop, script) -> None: - """Page text and run script against its keys while the pager is displayed.""" - created = [] - entered = threading.Event() - - def make_pager(*args, **kwargs): - pager = Pager(*args, **kwargs) - created.append(pager) - return pager - - def observe(ui): - if created and ui.full_screen and ui.layout.current_buffer is created[0].text.buffer: - entered.set() - - def interact(): - try: - assert entered.wait(5) - script(PagerKeys(app, pipe, created[0])) - finally: - pipe.send_text("q") - - app.main_session.app.after_render += observe - with mock.patch("cmd2.command_toolbar.Pager", side_effect=make_pager), ThreadPoolExecutor() as executor: - interaction = executor.submit(interact) - with app._command_toolbar_context(): - app._command_toolbar.page(text, chop=chop) - interaction.result(timeout=10) - assert get_typeahead(pipe) == [] - - -def test_pager_vertical_scrolling_keeps_horizontal_position(toolbar_app) -> None: - app, pipe, _ = toolbar_app - # Wide rows are what chopped output is for. Scrolling right to read a column and - # then moving down a row must not throw that column away. - text = "\n".join(f"row {index:03d} " + "col " * 40 for index in range(100)) - - def script(keys) -> None: - keys.press("\x1b[C", row=0, column=40) - keys.press("j", row=1, column=40) - keys.press("k", row=0, column=40) - - drive_pager(app, pipe, text, chop=True, script=script) - - -@pytest.mark.parametrize("chop", [False, True]) -def test_pager_navigation_keys(toolbar_app, chop) -> None: - app, pipe, _ = toolbar_app - lines = [f"row {index:03d}" for index in range(100)] - lines[3] = "" # A line with no columns for a scroll target to be clamped against. - - def script(keys) -> None: - keys.press("\x1b[B", row=1) # Down arrow. - keys.press("\x1b[A", row=0) # Up arrow. - page_rows = keys.pager.text.window.render_info.window_height - 1 - keys.press("\x1b[6~", row=page_rows) # Page Down. - keys.press("\x1b[5~", row=0) # Page Up. - keys.press("\r", row=1) - keys.press("\x1b[A", row=0) - keys.press("j", row=1) - keys.press("j", row=2) - keys.press("j", row=3) # Land on the empty line. - keys.press("k", row=2) # Moving up off it re-enters the line above. - keys.press("G", row=99) - keys.press("g", row=0) - keys.press("/row 05\n", row=50) - keys.press("n", row=51) - keys.press("N", row=50) - keys.press("?row 01\n", row=19) - keys.press("x", row=19) # Unbound keys are swallowed, not queued for the prompt. - # Horizontal scrolling applies only to chopped output, and stops at the - # end of a line shorter than the requested column. - keys.press("\x1b[C", row=19, column=len("row 019") if chop else 0) - keys.press("\x1b[D", row=19, column=0) - - drive_pager(app, pipe, "\n".join(lines), chop=chop, script=script) - - -@pytest.mark.parametrize("chop", [False, True]) -def test_pager_scrolling_before_first_render(chop) -> None: - pager = Pager("row\n" * 100, chop=chop) - # Keys can arrive before the first frame is drawn, when the window still has no - # rendered geometry to scroll against. - assert pager.text.window.render_info is None - event = SimpleNamespace(current_buffer=pager.text.buffer) - pager._scroll_page(event, pages=1.0) - pager._scroll(event, 1) - pager._scroll_horizontal(event, 1) - assert pager.text.buffer.cursor_position == 0 - assert pager.text.window.horizontal_scroll == 0 - - -@pytest.mark.parametrize("chop", [False, True]) -def test_pager_half_page_scrolling(toolbar_app, chop) -> None: - """Half-page keys move half of what the full-page keys move, and never zero rows.""" - app, pipe, _ = toolbar_app - lines = [f"row {index:03d}" for index in range(200)] - - def script(keys) -> None: - page_rows = max(1, keys.pager.text.window.render_info.window_height - 1) - half_rows = max(1, int(page_rows * 0.5)) - # A half page has to be a distinct, smaller step for this test to mean - # anything. The pager's own arithmetic is what decides the row it lands on. - assert 0 < half_rows < page_rows - keys.press("d", row=half_rows) - keys.press("d", row=2 * half_rows) - keys.press("u", row=half_rows) - keys.press("\x04", row=2 * half_rows) # Ctrl-D. - keys.press("\x15", row=half_rows) # Ctrl-U. - # Half-page steps stay half a page next to a full one taken from the same row. - keys.press("g", row=0) - keys.press("\x1b[6~", row=page_rows) # Page Down. - - drive_pager(app, pipe, "\n".join(lines), chop=chop, script=script) - - -@pytest.mark.parametrize("key", ["\x1b", "\x03"], ids=["escape", "ctrl-c"]) -def test_pager_search_abort_keys(toolbar_app, key) -> None: - """Escape and Ctrl-C abort a search rather than closing the pager. - - While the search field has focus the pager's own close bindings are filtered out, - so these keys have to reach prompt-toolkit's search bindings instead. Those are - registered explicitly so that they still work when the main prompt uses Vi mode. - """ - app, pipe, _ = toolbar_app - lines = [f"row {index:03d}" for index in range(100)] - - def script(keys) -> None: - keys.press("j", row=1) - # Typing a search previews it without moving the pager's own cursor. - keys.press("/row 05", row=1) - # Aborting restores the row the search started from instead of quitting. - keys.press(key, row=1) - # Focus is back on the pager, so ordinary navigation works again. - keys.press("j", row=2) - - drive_pager(app, pipe, "\n".join(lines), chop=False, script=script) - - -@pytest.mark.parametrize("key", ["\x1b", "\x03", "q"], ids=["escape", "ctrl-c", "q"]) -def test_pager_close_keys(toolbar_app, key) -> None: - """Each close key ends the pager without leaving the keystroke for the next prompt. - - Escape is bound eagerly, and it is also the first byte of every arrow and page - key. Closing on a bare Escape must therefore not come at the cost of the escape - sequences that arrive with more bytes behind them. - """ - app, pipe, _ = toolbar_app - entered = threading.Event() - closed = threading.Event() - - def observe(ui) -> None: - if ui.full_screen: - entered.set() - - def interact() -> None: - assert entered.wait(5), "pager never opened" - # Scroll first, so the pager is known to be reading keys before the close key. - pipe.send_text("j") - pipe.send_text(key) - if not closed.wait(5): - # Rescue the blocked main thread so this fails as an assertion, not a hang. - pipe.send_text("q") - raise AssertionError(f"{key!r} did not close the pager") - - app.main_session.app.after_render += observe - with ThreadPoolExecutor() as executor: - interaction = executor.submit(interact) - try: - with app._command_toolbar_context(): - app._command_toolbar.page("\n".join(f"row {index:03d}" for index in range(200)), chop=False) - finally: - closed.set() - interaction.result(timeout=10) - assert get_typeahead(pipe) == [] diff --git a/tests/test_pipeline_job_control.py b/tests/test_pipeline_job_control.py index 64dcfe890..2d2df6a05 100644 --- a/tests/test_pipeline_job_control.py +++ b/tests/test_pipeline_job_control.py @@ -631,18 +631,6 @@ def printing(screen): ) -def test_pager_resumes_with_raw_terminal_after_suspend(tmp_path) -> None: - """Ctrl-Z in the built-in pager, then fg, must bring the pager back, not echoed cursor reports.""" - suspend_and_resume( - tmp_path, - " def do_page(self, statement):\n self.ppaged('\\n'.join(f'line {n:04d}' for n in range(400)))\n", - "page", - lambda screen: any("q: quit" in line for line in screen.display), - # A cooked terminal would hold the key back until Enter: the pager must quit on q alone. - lambda send: send("q"), - ) - - @pytest.mark.parametrize("parent_delay", [False, True]) def test_pipeline_pager_can_set_terminal_modes_at_startup(tmp_path, parent_delay) -> None: """A pager such as less puts the terminal in raw mode as it starts, before reading its pipe. From b7dd0fd832fac65bd10504d666b4f40ea10dfc21 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt <2812720+tleonhardt@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:45:49 -0500 Subject: [PATCH 35/35] Describe the Textual toolbar example as an alternative to cmd2's own toolbar The example and its documentation came from main, where the bottom toolbar is only on the screen while the prompt is. They said so, and that cmd2 does not keep a toolbar there while a command runs. On this branch it does. Say what the example still offers: a toolbar drawn outside the terminal the cmd2 application runs in, which therefore stays when something takes over that terminal, as a shell command does, where cmd2 has to hide its own. --- docs/examples/embedding_in_textual.md | 17 +++++++++++------ examples/textual_tty_toolbar.py | 19 ++++++++++--------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/docs/examples/embedding_in_textual.md b/docs/examples/embedding_in_textual.md index 25924bc83..e1d4c6ada 100644 --- a/docs/examples/embedding_in_textual.md +++ b/docs/examples/embedding_in_textual.md @@ -120,9 +120,12 @@ The example handles several details which are easy to miss when building somethi ## A Bottom Toolbar Which Stays While Commands Run -A [bottom toolbar](../features/prompt.md#bottom-toolbar) is drawn by `prompt-toolkit` as part of the -prompt, so it is only on the screen while the prompt is. The same embedding technique gives you one -which stays, and keeps refreshing, while a command runs. The +`cmd2` can keep its own [bottom toolbar](../features/prompt.md#bottom-toolbar) on the screen while a +command runs. It draws that toolbar in the terminal which the command is using, so it has to hide it +whenever something else +[takes over the terminal](../features/prompt.md#commands-that-take-over-the-terminal), as a shell +command does. The same embedding technique gives you a toolbar which is drawn outside that terminal, +so it stays, and keeps refreshing, whatever runs there. The [textual_tty_toolbar.py](https://github.com/python-cmd2/cmd2/blob/main/examples/textual_tty_toolbar.py) example is a Textual application which is nothing but a terminal pane running a `cmd2` application and a toolbar docked below it, so it looks like the `cmd2` application running on its own: @@ -131,8 +134,9 @@ and a toolbar docked below it, so it looks like the `cmd2` application running o uv run examples/textual_tty_toolbar.py ``` -Try `slow`, `burst 2000 1`, or resizing the window while a command runs. Pass `--cmd2-only` to see -what `cmd2` does without Textual, where the toolbar disappears for as long as a command runs. +Try `slow`, `burst 2000 1`, `shell sleep 5`, or resizing the window while a command runs. Pass +`--cmd2-only` to compare it with the toolbar `cmd2` draws itself, without Textual, which is hidden +for as long as the shell command runs. Here the `cmd2` application doesn't draw the toolbar at all. It passes `enable_bottom_toolbar=False` to [cmd2.Cmd.__init__][] when it is running inside the Textual application, and sends what @@ -166,7 +170,8 @@ in the middle of the command's output. So this example uses a Unix socket instea - `get_bottom_toolbar()` is therefore called on that thread while a command is running on the main thread, so it must only read state which is safe to read from another thread. -This approach has costs which are easy to overlook, and they are why `cmd2` doesn't do it for you: +This approach has costs which are easy to overlook, and they are why `cmd2` draws its own toolbar in +the user's terminal instead: - There is no scrollback. The terminal pane keeps none, so output which scrolls off its top is gone, and Textual uses the terminal's alternate screen, so none of the session is in the scrollback of diff --git a/examples/textual_tty_toolbar.py b/examples/textual_tty_toolbar.py index 7c46e760f..a2b40348c 100755 --- a/examples/textual_tty_toolbar.py +++ b/examples/textual_tty_toolbar.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 """Keep a bottom toolbar on the screen while commands run, by letting Textual draw it using textual-tty. -cmd2's bottom toolbar is drawn by prompt-toolkit as part of the prompt, so it is only there while the prompt is. -This example shows a way to have one which stays, and keeps refreshing, while a command runs: the cmd2 -application doesn't draw the toolbar at all. A small Textual application owns the terminal instead. It shows the -cmd2 application in a terminal pane which fills the screen except for the bottom, where it draws the toolbar. +cmd2 can keep its own bottom toolbar on the screen while a command runs. It draws that toolbar in the terminal +which the command is using, so it has to hide it whenever something else takes over the terminal, as a shell +command does. This example shows a way to have one which stays, and keeps refreshing, whatever runs there: the +cmd2 application doesn't draw the toolbar at all. A small Textual application owns the terminal instead. It shows +the cmd2 application in a terminal pane which fills the screen except for the bottom, where it draws the toolbar. How it works: - [textual-tty](https://github.com/ttygroup/textual-tty) provides the ``Terminal`` widget, a terminal @@ -22,7 +23,7 @@ output. A toolbar which refreshes during a command is sent from another thread, and a window title written into the terminal's stream from another thread could land in the middle of the command's output. -Things this approach gives up, which is why cmd2 doesn't do it for you: +Things this approach gives up, which is why cmd2 draws its own toolbar in your terminal instead: - Scrollback. The terminal pane keeps none, so output which scrolls off its top is gone, and Textual uses the terminal's alternate screen, so none of the session is in your terminal's own scrollback either. Pipe long output to a pager. @@ -38,8 +39,9 @@ Otherwise, install textual-tty into your own environment with ``pip install textual-tty`` and then run ``python examples/textual_tty_toolbar.py``. -Things to try: ``slow``, ``burst 2000 1``, ``page``, ``ask``, resizing the window while a command runs, and -Ctrl-C during ``slow``. To compare with what cmd2 does on its own, run just the cmd2 application, on any platform: +Things to try: ``slow``, ``burst 2000 1``, ``page``, ``ask``, ``shell sleep 5``, resizing the window while a +command runs, and Ctrl-C during ``slow``. To compare with the toolbar cmd2 draws itself, run just the cmd2 +application, on any platform: python examples/textual_tty_toolbar.py --cmd2-only @@ -171,8 +173,7 @@ def __init__(self, toolbar_socket: str | None) -> None: """Initialize the cmd2 application. :param toolbar_socket: the Unix socket of the Textual application which draws the toolbar, or None if this - application isn't running inside one. Then prompt-toolkit draws the toolbar as usual, - which means only while the prompt is on the screen. + application isn't running inside one. Then cmd2 draws the toolbar itself. """ refresh_interval = 0.5 super().__init__(