Set the Windows console to UTF-8 while a pipe or the pager runs

Pipes used the console's code page so that console programs such as more, sort and findstr
would show them correctly. But UTF-8 tools, such as rg or Git for Windows, then lost every
character the code page lacks, and piped output differed from redirected output.

Pipes are UTF-8 again everywhere. On Windows with a console, cmd2 sets its input and output code
pages to 65001 from before the pipe process or pager starts until it exits, as chcp 65001 would,
so console programs decode that UTF-8 too. The code pages it replaced are restored once the last
overlapping pipe ends, including when the pipe fails.

Author: tleonhardt

CHANGELOG.md

@@ -11,11 +11,10 @@
     - A `shell` command piped to an interactive program, such as `shell git log | less`, now joins
       the pipeline's job, so both processes receive Ctrl-C and Ctrl-Z
     - On Windows, fixed piped output appearing garbled in console programs such as `more`
-      (`help -v | more`), a regression in 4.2.4. Pipes were written as UTF-8, but console programs
-      decode their input with the console's code page. Pipes now use that code page, including ones
-      Python names otherwise, such as 20866 (KOI8-R) or 28591 (ISO-8859-1), and characters it cannot
-      represent are replaced rather than failing the command. Without a console, with a code page
-      Python has no codec for, and on other platforms, pipes still use UTF-8
+      (`help -v | more`), a regression in 4.2.4. Pipes are written as UTF-8, but console programs
+      decode their input with the console's code page. While a pipe or the pager runs, cmd2 now sets
+      the console to UTF-8 (code page 65001, as `chcp 65001` does) and restores its code pages
+      afterward, so console programs and UTF-8 tools such as `rg` both read the output correctly
     - When copying a command's output to the clipboard (`help >`) fails, later commands' output is
       no longer written to the temporary file that held it
 

cmd2/cmd2.py

@@ -1965,12 +1965,12 @@ def ppaged(
                     soft_wrap=soft_wrap,
                     **(rich_print_kwargs if rich_print_kwargs is not None else {}),
                 )
-            # As for a pipe: on Windows, the pager decodes with the console's code page.
-            output_bytes = capture.get().encode(utils._pipe_encoding(), "replace")
+            # As for a pipe: the pager gets UTF-8, and on Windows the console is UTF-8 while it runs.
+            output_bytes = capture.get().encode("utf-8", "replace")
 
             # Prevent KeyboardInterrupts while in the pager. The pager application will
             # still receive the SIGINT since it is in the same process group as us.
-            with self.sigint_protection:
+            with self.sigint_protection, utils._utf8_console():
                 import subprocess
 
                 pipe_proc = subprocess.Popen(  # noqa: S602
@@ -3320,13 +3320,11 @@ def _redirect_output(self, statement: Statement) -> utils.RedirectionSavedState:
             # Create a pipe with read and write sides
             read_fd, write_fd = os.pipe()
 
-            # Open each side of the pipe. Both ends are given an explicit encoding: command output is rendered by Rich and
-            # routinely contains non-ASCII, which the locale encoding cannot always represent. On Windows, that is the
-            # console's code page, which console programs such as more decode with. It cannot represent everything either, so
-            # replace what it lacks rather than fail the command.
-            pipe_encoding = utils._pipe_encoding()
-            subproc_stdin = open(read_fd, encoding=pipe_encoding)  # noqa: SIM115
-            new_stdout: TextIO = cast(TextIO, open(write_fd, "w", encoding=pipe_encoding, errors="replace"))  # noqa: SIM115
+            # Open each side of the pipe. Both ends use UTF-8 rather than the locale's encoding, which cannot always represent
+            # the non-ASCII that Rich routinely renders. On Windows, the console is UTF-8 too while the pipe runs, for console
+            # programs such as more, which decode with its code page.
+            subproc_stdin = open(read_fd, encoding="utf-8")  # noqa: SIM115
+            new_stdout: TextIO = cast(TextIO, open(write_fd, "w", encoding="utf-8"))  # noqa: SIM115
 
             # Isolate pipeline signals from cmd2. Terminal pipelines receive the foreground terminal; ProcReader relays their
             # job-control stops.
@@ -3373,7 +3371,10 @@ def _redirect_output(self, statement: Statement) -> utils.RedirectionSavedState:
                     popen_command = f"read -r _ || exit 1; exec {user_shell} -c {shlex.quote(statement.redirect_to)}"
                     kwargs["executable"] = posix_shell
 
-            with contextlib.ExitStack() as terminal_stack, contextlib.ExitStack() as gate_stack:
+            with contextlib.ExitStack() as pipe_stack, contextlib.ExitStack() as gate_stack:
+                # The console stays UTF-8 from before the pipe process starts, since a program may read the code page once as
+                # it starts, until _restore_output() has reaped it.
+                pipe_stack.enter_context(utils._utf8_console())
                 if terminal_fd is not None:
                     # Should cmd2 fail before opening the gate, the held pipeline reads EOF and exits.
                     gate_stack.callback(new_stdout.close)
@@ -3391,7 +3392,7 @@ def _redirect_output(self, statement: Statement) -> utils.RedirectionSavedState:
                 subproc_stdin.close()
                 if terminal_fd is not None:
                     cmd_pipe_proc_reader = utils.ProcReader(proc, self.stdout, sys.stderr, terminal_fd=terminal_fd)
-                    terminal_stack.enter_context(cmd_pipe_proc_reader._manage_terminal())
+                    pipe_stack.enter_context(cmd_pipe_proc_reader._manage_terminal())
 
                 # Popen was called with shell=True so the user can chain pipe commands and redirect their output
                 # like: !ls -l | grep user | wc -l > out.txt. But this makes it difficult to know if the pipe process started
@@ -3429,14 +3430,13 @@ def _redirect_output(self, statement: Statement) -> utils.RedirectionSavedState:
                                 pipe_fd, cmd_pipe_proc_reader, interruptible=lambda: not self.sigint_protection
                             )
                         ),
-                        encoding=pipe_encoding,
-                        errors="replace",
+                        encoding="utf-8",
                     )
 
                 self.stdout = new_stdout
 
-                # Keep the pipeline's job control until _restore_output() reaps the pipe process.
-                redir_saved_state.pipeline_job = terminal_stack.pop_all()
+                # Keep the UTF-8 console and the pipeline's job control until _restore_output() reaps the pipe process.
+                redir_saved_state.pipe_context = pipe_stack.pop_all()
 
         elif statement.redirector in (constants.REDIRECTION_OVERWRITE, constants.REDIRECTION_APPEND):
             if statement.redirect_to:
@@ -3495,11 +3495,11 @@ def _restore_output(self, statement: Statement, saved_redir_state: utils.Redirec
         :param statement: Statement object which contains the parsed input from the user
         :param saved_redir_state: contains information needed to restore state data
         """
-        # The pipeline's job control ends once its pipe process has been reaped.
-        with contextlib.ExitStack() as terminal_stack:
-            if saved_redir_state.pipeline_job is not None:
-                terminal_stack.callback(saved_redir_state.pipeline_job.close)
-                saved_redir_state.pipeline_job = None
+        # The UTF-8 console and the pipeline's job control end once its pipe process has been reaped.
+        with contextlib.ExitStack() as pipe_stack:
+            if saved_redir_state.pipe_context is not None:
+                pipe_stack.callback(saved_redir_state.pipe_context.close)
+                saved_redir_state.pipe_context = None
 
             try:
                 if saved_redir_state.redirecting:

cmd2/utils.py

@@ -536,58 +536,62 @@ def write(self, b: bytes) -> None:
                 self.std_sim_instance.flush()
 
 
-# Windows code pages Python names other than cpNNNNN. Before Python 3.14, which covers every code page Windows supports, a
-# console set to one of these would otherwise get UTF-8.
-_CODE_PAGE_CODECS = {
-    20127: "ascii",
-    20866: "koi8_r",
-    21866: "koi8_u",
-    28591: "latin_1",
-    28592: "iso8859_2",
-    28593: "iso8859_3",
-    28594: "iso8859_4",
-    28595: "iso8859_5",
-    28596: "iso8859_6",
-    28597: "iso8859_7",
-    28598: "iso8859_8",
-    28599: "iso8859_9",
-    28603: "iso8859_13",
-    28605: "iso8859_15",
-    51932: "euc_jp",
-    51949: "euc_kr",
-    54936: "gb18030",
-}
-
-
-def _code_page_encoding(code_page: int) -> str | None:
-    """Return the name of the Python codec for a Windows code page, or None if there is none.
-
-    :param code_page: a Windows code page identifier, such as 437
+class _Utf8Console:
+    """Set the Windows console to UTF-8 while cmd2 pipes output to a shell command or pager.
+
+    cmd2 writes pipes as UTF-8. Windows console programs such as more, sort, and findstr decode
+    piped input with the console's output code page, PowerShell with its input code page, and
+    tools such as rg with UTF-8. The UTF-8 code page, 65001, suits them all, as `chcp 65001`
+    would. It belongs to the whole console, so the code pages it replaced are restored once the
+    last pipe that uses it has ended. Without a console, and on other platforms, it does nothing.
     """
-    import codecs
 
-    for name in (f"cp{code_page}", _CODE_PAGE_CODECS.get(code_page)):
-        if name is not None:
-            with contextlib.suppress(LookupError):
-                # Normalized, so that code page 65001 is reported as utf-8
-                return codecs.lookup(name).name
-    return None
+    UTF8 = 65001
 
+    def __init__(self, kernel32: Any = None) -> None:
+        """Initialize for the console of a Windows API.
 
-def _pipe_encoding() -> str:
-    """Return the encoding for output cmd2 pipes to a shell command.
+        :param kernel32: the Windows API to use, or None for ctypes.windll.kernel32 on Windows
+        """
+        self._kernel32 = kernel32
+        self._lock = threading.Lock()
+        # How many pipes use the console, and its input and output code pages to restore after the last
+        self._users = 0
+        self._saved: tuple[int, int] | None = None
+
+    @contextlib.contextmanager
+    def __call__(self) -> Iterator[None]:
+        """Keep the console at UTF-8 for as long as the context lasts."""
+        kernel32 = self._kernel32
+        if kernel32 is None and sys.platform == "win32":
+            import ctypes
+
+            kernel32 = ctypes.windll.kernel32
+        if kernel32 is None:
+            yield
+            return
+        with self._lock:
+            if not self._users:
+                # Without a console, the code pages are 0.
+                code_pages = (kernel32.GetConsoleCP(), kernel32.GetConsoleOutputCP())
+                if code_pages[1] and code_pages != (self.UTF8, self.UTF8):
+                    self._saved = code_pages
+                    kernel32.SetConsoleCP(self.UTF8)
+                    kernel32.SetConsoleOutputCP(self.UTF8)
+            self._users += 1
+        try:
+            yield
+        finally:
+            with self._lock:
+                self._users -= 1
+                if not self._users and self._saved is not None:
+                    input_code_page, output_code_page = self._saved
+                    self._saved = None
+                    kernel32.SetConsoleCP(input_code_page)
+                    kernel32.SetConsoleOutputCP(output_code_page)
 
-    Windows console programs such as more, sort, and findstr decode piped input with the
-    console's output code page, and would show UTF-8 as mojibake. Elsewhere, and on Windows
-    without a console or with a code page Python cannot encode, use UTF-8.
-    """
-    if sys.platform == "win32":
-        import ctypes
 
-        code_page = ctypes.windll.kernel32.GetConsoleOutputCP()
-        if code_page:
-            return _code_page_encoding(code_page) or "utf-8"
-    return "utf-8"
+_utf8_console = _Utf8Console()
 
 
 @contextlib.contextmanager
@@ -1460,8 +1464,9 @@ def __init__(
         self.saved_pipe_proc_reader = pipe_proc_reader
         self.saved_redirecting = saved_redirecting
 
-        # Holds a terminal pipeline's job control until its pipe process has been reaped
-        self.pipeline_job: contextlib.ExitStack | None = None
+        # Holds what a pipe needs until its pipe process has been reaped: on Windows a UTF-8 console, and for a terminal
+        # pipeline its job control
+        self.pipe_context: contextlib.ExitStack | None = None
 
 
 def categorize(func: Callable[..., Any] | Iterable[Callable[..., Any]], category: str) -> None:

tests/test_cmd2.py

@@ -3795,8 +3795,11 @@ def test_ppaged_with_pager(outsim_app, monkeypatch, chop) -> None:
     assert expected_cmd == popen_mock.call_args_list[0].args[0]
 
 
-def test_ppaged_encodes_for_the_console(outsim_app, monkeypatch) -> None:
-    """As for a pipe: on Windows, the pager decodes with the console's code page, and what it lacks is replaced."""
+def test_ppaged_pages_utf8_in_a_utf8_console(outsim_app, monkeypatch) -> None:
+    """As for a pipe: the pager gets UTF-8, and on Windows the console is UTF-8 until the pager has exited.
+
+    The pager there is more, which decodes with the console's code page.
+    """
     stdin_mock = mock.MagicMock()
     stdin_mock.isatty.return_value = True
     monkeypatch.setattr(outsim_app, "stdin", stdin_mock)
@@ -3805,12 +3808,25 @@ def test_ppaged_encodes_for_the_console(outsim_app, monkeypatch) -> None:
     monkeypatch.setattr(outsim_app, "stdout", stdout_mock)
     if not sys.platform.startswith("win") and os.environ.get("TERM") is None:
         monkeypatch.setenv("TERM", "simulated")
-    monkeypatch.setattr("cmd2.utils._pipe_encoding", lambda: "cp437")
+    events = []
+
+    @contextlib.contextmanager
+    def utf8_console():
+        events.append("utf-8 console")
+        try:
+            yield
+        finally:
+            events.append("restored")
+
+    monkeypatch.setattr("cmd2.utils._utf8_console", utf8_console)
     popen_mock = mock.MagicMock(name="Popen")
+    popen_mock.side_effect = lambda *args, **kwargs: events.append("pager started") or popen_mock.return_value
+    popen_mock.return_value.communicate.side_effect = lambda *args: events.append("pager exited")
     monkeypatch.setattr("subprocess.Popen", popen_mock)
     outsim_app.ppaged("box ─ smile \U0001f642")
     paged = popen_mock.return_value.communicate.call_args.args[0]
-    assert "box ─ smile ?".encode("cp437") in paged
+    assert "box ─ smile \U0001f642".encode() in paged
+    assert events == ["utf-8 console", "pager started", "pager exited", "restored"]
 
 
 def test_ppaged_no_pager(outsim_app) -> None:

tests/test_suite_environment.py

@@ -67,27 +67,56 @@ def test_redirection_to_a_file_uses_utf8(tmp_path) -> None:
 PASS_THROUGH = "import sys; sys.stdin.reconfigure(encoding='utf-8'); sys.stdout.write(sys.stdin.read())"
 
 
-def test_piping_uses_pipe_encoding(tmp_path, running_pipe_process) -> None:
-    """The pipe the subprocess reads from uses the encoding its consumer expects, not the locale's.
+def test_piping_uses_utf8(tmp_path, running_pipe_process) -> None:
+    """The pipe the subprocess reads from uses UTF-8, not the locale's encoding.
 
-    That is UTF-8, except on Windows, where console programs such as more decode with the
-    console's code page.
+    On Windows, console programs such as more decode with the console's code page, so cmd2
+    sets the console to UTF-8 while the pipe runs.
     """
     app = EncodingProbe(allow_cli_args=False)
     target = tmp_path / "piped.txt"
     app.onecmd_plus_hooks(f'show_encoding | "{sys.executable}" -c "{PASS_THROUGH}" > "{target}"')
-    assert f"ENCODING={cmd2.utils._pipe_encoding()}" in target.read_text(encoding="utf-8")
+    assert "ENCODING=utf-8" in target.read_text(encoding="utf-8")
+
+
+def test_piping_sets_the_console_to_utf8_while_the_pipe_runs(tmp_path, monkeypatch, running_pipe_process) -> None:
+    """The console is UTF-8 before the pipe process starts, and stays so until it has exited."""
+    import contextlib
+    import subprocess
+
+    events = []
+    processes = []
+    real_popen = subprocess.Popen
+
+    @contextlib.contextmanager
+    def utf8_console():
+        events.append("utf-8 console")
+        try:
+            yield
+        finally:
+            events.append(("restored", [process.returncode for process in processes]))
+
+    def popen(*args, **kwargs):
+        events.append("pipe started")
+        processes.append(real_popen(*args, **kwargs))
+        return processes[-1]
+
+    monkeypatch.setattr(cmd2.utils, "_utf8_console", utf8_console)
+    monkeypatch.setattr(subprocess, "Popen", popen)
+    app = EncodingProbe(allow_cli_args=False)
+    target = tmp_path / "piped.txt"
+    app.onecmd_plus_hooks(f'say hello | "{sys.executable}" -c "{PASS_THROUGH}" > "{target}"')
+    assert events == ["utf-8 console", "pipe started", ("restored", [0])]
+    assert "hello" in target.read_text(encoding="utf-8")
 
 
 #: A pass-through filter that copies bytes, so the test sees exactly what cmd2 wrote to the pipe.
 BYTES_PASS_THROUGH = "import sys; sys.stdout.buffer.write(sys.stdin.buffer.read())"
 
 
-def test_piping_replaces_what_the_pipe_encoding_lacks(tmp_path, monkeypatch, running_pipe_process) -> None:
-    """A console code page cannot represent all output, which must not fail the command."""
-    monkeypatch.setattr(cmd2.utils, "_pipe_encoding", lambda: "cp437")
+def test_piping_keeps_all_output(tmp_path, running_pipe_process) -> None:
+    """UTF-8 represents all output, such as box drawing and emoji, which a console code page could not."""
     app = EncodingProbe(allow_cli_args=False)
     target = tmp_path / "piped.bin"
     app.onecmd_plus_hooks(f'say ─\U0001f607 | "{sys.executable}" -c "{BYTES_PASS_THROUGH}" > "{target}"')
-    # Box drawing exists in cp437, the emoji does not
-    assert target.read_bytes().startswith(b"\xc4?")
+    assert target.read_bytes().startswith("─\U0001f607".encode())