Skip to content
Merged
15 changes: 15 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,21 @@
- Bug Fixes
- Fixed file path completion for names containing square brackets, including bracketed working
directories and home directories.
- Fixed `get_exes_in_path()` and shell command completion when executable names or `PATH`
directories contain square brackets.
- File path completion, shell command completion, and `get_exes_in_path()` now match `*` and `?`
literally instead of returning nothing when the text being completed contains one, so names
containing those characters can be completed.
- Tab completion now adds quotes when a match has a character which is special to a shell, such
as `*`, `?`, `(`, `&`, or `$`, if what is being completed is passed to one. That is the case
for the arguments of the `shell` command and for everything after a pipe. Everywhere else,
only a match with a space is quoted, as before. See the
[Special Characters in Paths](docs/features/completion.md#special-characters-in-paths)
documentation for details.
- `get_exes_in_path()`, `find_editor()`, and shell command completion no longer skip `PATH`
entries which are symbolic links to directories.
- `find_editor()` now finds the `edit` editor on Windows. It was the preferred default editor
there but could never be found because it was searched for without its `.exe` extension.
- A command which prints through a `cmd2` console of its own, such as a `Cmd2GeneralConsole`
showing a Rich progress bar, no longer ends with a `BrokenPipeError` message when its output
is piped to something that exits early, as `head` does. `cmd2` now ends the command quietly
Expand Down
62 changes: 43 additions & 19 deletions cmd2/cmd2.py
Original file line number Diff line number Diff line change
Expand Up @@ -2385,6 +2385,10 @@ def path_complete(
) -> Completions:
"""Perform completion of local file system paths.

Apart from a leading ``~``, which refers to a user's home directory, every character in ``text`` is
matched literally. Glob characters like ``*``, ``?``, and ``[`` are not expanded, and they are not
escaped in the results.

:param text: the string prefix we are attempting to match (all matches must begin with it)
:param line: the current input line with leading whitespace removed
:param begidx: the beginning index of the prefix text
Expand All @@ -2399,25 +2403,19 @@ def path_complete(
if endidx == len(line) or (endidx < len(line) and line[endidx] != os.path.sep):
add_trailing_sep_if_dir = True

# Used to replace cwd in the final results
# Used to replace CWD in the final results
cwd = os.getcwd()
cwd_added = False

# Used to replace expanded user path in final result
orig_tilde_path = ""
expanded_tilde_path = ""

# If the search text is blank, then search in the CWD for *
# If the search text is blank, then complete all entries in the CWD
if not text:
search_str = os.path.join(os.getcwd(), "")
cwd_added = True
else:
# Purposely don't match any path containing wildcards
wildcards = ["*", "?"]
for wildcard in wildcards:
if wildcard in text:
return Completions()

# Start the search string
search_str = text

Expand All @@ -2436,7 +2434,7 @@ def path_complete(
orig_tilde_path = text[:sep_index]
expanded_tilde_path = os.path.expanduser(orig_tilde_path)

# If the search text does not have a directory, then use the cwd
# If the search text does not have a directory, then use the CWD
elif not os.path.dirname(text):
search_str = os.path.join(os.getcwd(), search_str)
cwd_added = True
Expand Down Expand Up @@ -2465,7 +2463,7 @@ def path_complete(
matches[index] += os.path.sep
display_matches[index] += os.path.sep

# Remove cwd if it was added to match the text prompt-toolkit expects
# Remove CWD if it was added to match the text prompt-toolkit expects
if cwd_added:
to_replace = cwd if cwd == os.path.sep else cwd + os.path.sep
matches = [cur_path.replace(to_replace, "", 1) for cur_path in matches]
Expand Down Expand Up @@ -2501,7 +2499,7 @@ def shell_cmd_complete(
if not complete_blank and not text:
return Completions()

# If there are no path characters in the search text, then do shell command completion in the user's path
# If the search text is a bare command name (no '~' or path separator), complete executables from PATH
if not text.startswith("~") and os.path.sep not in text:
items = [CompletionItem(exe) for exe in utils.get_exes_in_path(text)]
return Completions(items=items)
Expand Down Expand Up @@ -2536,6 +2534,9 @@ def _redirect_complete(self, text: str, line: str, begidx: int, endidx: int, com
# True when command line contains any redirection tokens
has_redirection = False

# True when the first redirection token is a pipe, which means everything after it is passed to a shell
piped_to_shell = False

# Keep track of state while examining tokens
in_pipe = False
in_file_redir = False
Expand All @@ -2546,6 +2547,8 @@ def _redirect_complete(self, text: str, line: str, begidx: int, endidx: int, com
for cur_token in raw_tokens:
# Process redirection tokens
if cur_token in constants.REDIRECTION_TOKENS:
if not has_redirection:
piped_to_shell = cur_token == constants.REDIRECTION_PIPE
has_redirection = True

# Check if we are at a pipe
Expand Down Expand Up @@ -2578,11 +2581,10 @@ def _redirect_complete(self, text: str, line: str, begidx: int, endidx: int, com

prior_token = cur_token

if do_shell_completion:
return self.shell_cmd_complete(text, line, begidx, endidx)

if do_path_completion:
return self.path_complete(text, line, begidx, endidx)
if do_shell_completion or do_path_completion:
completer = self.shell_cmd_complete if do_shell_completion else self.path_complete
completions = completer(text, line, begidx, endidx)
return self._for_shell(completions) if piped_to_shell else completions

# If there were redirection strings anywhere on the command line, then we
# are no longer completing for the current command
Expand Down Expand Up @@ -2729,7 +2731,13 @@ def _perform_completion(
if not completion_token_quote:
matches = completions.to_strings()

if any(" " in match for match in matches):
# Matches which are being passed to a shell need quotes if any of them has a character special to it
if completions._quote_for_shell:
_quote_char = utils._shell_quote_char(matches)
_add_opening_quote = bool(_quote_char)

# Everything else only needs quotes to keep a match with a space as a single token
elif any(" " in match for match in matches):
_add_opening_quote = True

# Determine best quote (single vs double) based on text content
Expand Down Expand Up @@ -5036,13 +5044,29 @@ def do_set(self, args: argparse.Namespace) -> None:
@classmethod
def _build_shell_parser(cls) -> Cmd2ArgumentParser:
shell_parser = argparse_utils.DEFAULT_ARGUMENT_PARSER(description="Execute a command as if at the OS prompt.")
shell_parser.add_argument("command", help="the command to run", completer=cls.shell_cmd_complete)
shell_parser.add_argument("command", help="the command to run", completer=cls._complete_shell_command)
shell_parser.add_argument(
"command_args", nargs=argparse.REMAINDER, help="arguments to pass to command", completer=cls.path_complete
"command_args", nargs=argparse.REMAINDER, help="arguments to pass to command", completer=cls._complete_shell_arg
)

return shell_parser

@staticmethod
def _for_shell(completions: Completions) -> Completions:
"""Mark completions as being for part of the command line which is passed to a shell.

This is what gets them quoted when one of them has a character which is special to that shell.
"""
return dataclasses.replace(completions, _quote_for_shell=True)

def _complete_shell_command(self, text: str, line: str, begidx: int, endidx: int) -> Completions:
"""Complete the command which do_shell passes to a shell."""
return self._for_shell(self.shell_cmd_complete(text, line, begidx, endidx))

def _complete_shell_arg(self, text: str, line: str, begidx: int, endidx: int) -> Completions:
"""Complete an argument of the command which do_shell passes to a shell."""
return self._for_shell(self.path_complete(text, line, begidx, endidx))

# Preserve quotes since we are passing these strings to the shell
@with_argparser(_build_shell_parser, preserve_quotes=True)
def do_shell(self, args: argparse.Namespace) -> None:
Expand Down
4 changes: 4 additions & 0 deletions cmd2/completion.py
Original file line number Diff line number Diff line change
Expand Up @@ -305,3 +305,7 @@ class Completions(CompletionResultsBase):

# The quote character to use if adding an opening or closing quote to the matches.
_quote_char: str = ""

# Whether the matches are for part of the command line which is passed to a shell. If they are,
# then quotes are also added when a match has a character which is special to that shell.
_quote_for_shell: bool = field(default=False, compare=False)
72 changes: 54 additions & 18 deletions cmd2/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
Generator,
Iterable,
MutableSequence,
Sequence,
)
from difflib import SequenceMatcher
from typing import (
Expand Down Expand Up @@ -323,6 +324,36 @@ def expand_user_in_tokens(tokens: MutableSequence[str]) -> None:
tokens[index] = expand_user(tokens[index])


# Matches a character which is special to a POSIX shell. A '[' counts even when no ']' follows it since
# some shells, like zsh, reject that as a bad pattern. A '#' and a '=' only count at the start of a word.
_POSIX_SPECIAL_RE = re.compile(r"""[ \t"';|<>&()*?\[{}$`\\]|^[#=]""")

# Matches a character which cmd.exe documents as requiring quotes. A '~' is left out because it is not
# special to cmd.exe itself, and quoting it would quote every path below a home directory.
_WINDOWS_SPECIAL_RE = re.compile(r"""[ \t&()\[\]{}^=;!'+,`]""")


def _shell_quote_char(matches: Sequence[str]) -> str:
"""Return the quote character which completions being passed to a shell need to be wrapped in.

:param matches: the completions being offered for part of a command line which is passed to a shell
:return: the quote character to use, or an empty string if none of the matches need quotes
"""
if sys.platform.startswith("win"):
# cmd.exe only treats double quotes as quotes, and a Windows file name can't contain one
return '"' if any(_WINDOWS_SPECIAL_RE.search(match) for match in matches) else ""

if not any(_POSIX_SPECIAL_RE.search(match) for match in matches):
return ""

# A quote character within a match has to be quoted with the other quote character
if any("'" in match for match in matches):
return '"'

# Only use single quotes when a shell would still expand something within double quotes
return "'" if any(char in match for match in matches for char in '"$`\\') else '"'


def _quote_shell_arg(arg: str) -> str:
"""Quote a string so the OS shell treats it as one literal argument.

Expand All @@ -345,6 +376,21 @@ def _quote_shell_arg(arg: str) -> str:
return shlex.quote(arg)


def _get_path_dirs() -> set[str]:
"""Return the directories listed in the PATH environment variable.

The entries are returned as they are written. Resolving them would avoid searching a directory twice
when it is listed under two names, but doing so can raise an exception for an entry which can't be reached.

An empty entry is kept because a POSIX shell searches the current working directory for one. A PATH which
is set to an empty string is a single empty entry, so it is not the same as a PATH which is not set.

:return: the directories, which is an empty set if PATH is not set
"""
env_path = os.getenv("PATH")
return set() if env_path is None else set(env_path.split(os.path.pathsep))


def find_editor() -> str | None:
"""Set cmd2.Cmd.DEFAULT_EDITOR. If EDITOR env variable is set, that will be used.

Expand All @@ -354,15 +400,12 @@ def find_editor() -> str | None:
editor = os.environ.get("EDITOR")
if not editor:
if sys.platform[:3] == "win":
editors = ["edit", "code.cmd", "notepad++.exe", "notepad.exe"]
editors = ["edit.exe", "code.cmd", "notepad++.exe", "notepad.exe"]
else:
editors = ["vim", "vi", "emacs", "nano", "pico", "joe", "code", "subl", "gedit", "kate"]

# Get a list of every directory in the PATH environment variable and ignore symbolic links
env_path = os.getenv("PATH")
paths = [] if env_path is None else [p for p in env_path.split(os.path.pathsep) if not os.path.islink(p)]

for possible_editor, path in itertools.product(editors, paths):
# Use the most preferred editor which is in any of the PATH directories
for possible_editor, path in itertools.product(editors, _get_path_dirs()):
editor_path = os.path.join(path, possible_editor)
if os.path.isfile(editor_path) and os.access(editor_path, os.X_OK):
if sys.platform[:3] == "win":
Expand Down Expand Up @@ -408,26 +451,19 @@ def files_from_glob_patterns(patterns: Iterable[str], access: int = os.F_OK) ->
def get_exes_in_path(starts_with: str) -> list[str]:
"""Return names of executables in a user's path.

Every character in ``starts_with`` is matched literally. Glob characters like ``*``, ``?``, and ``[``
are not expanded.

:param starts_with: what the exes should start with. leave blank for all exes in path.
:return: a list of matching exe names
"""
# Purposely don't match any executable containing wildcards
wildcards = ["*", "?"]
for wildcard in wildcards:
if wildcard in starts_with:
return []

# Get a list of every directory in the PATH environment variable and ignore symbolic links
env_path = os.getenv("PATH")
paths = [] if env_path is None else [p for p in env_path.split(os.path.pathsep) if not os.path.islink(p)]

# Use a set to store exe names since there can be duplicates
exes_set = set()

# Find every executable file in the user's path that matches the pattern
for path in paths:
for path in _get_path_dirs():
full_path = os.path.join(path, starts_with)
matches = files_from_glob_pattern(full_path + "*", access=os.X_OK)
matches = files_from_glob_pattern(glob.escape(full_path) + "*", access=os.X_OK)
Comment thread
tleonhardt marked this conversation as resolved.
Resolved

for match in matches:
exes_set.add(os.path.basename(match))
Expand Down
46 changes: 46 additions & 0 deletions docs/features/completion.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,52 @@ line of code similar to the following to your class which inherits from `cmd2.Cm
complete_bar = functools.partialmethod(cmd2.Cmd.path_complete, path_filter=os.path.isdir)
```

## Special Characters in Paths

Path and shell command completion match the text being completed literally. Glob characters such as
`*`, `?`, and `[` are never expanded, so a file named `my*file.txt` or `notes[2026].txt` completes
like any other. The one exception is a leading `~`, which refers to a user's home directory as it
does in a shell.

Completion results are literal names too. `cmd2` does not escape them, because most commands use a
path directly in Python, where the literal name is exactly what is wanted. The only thing `cmd2`
adds is quotes around a match which contains a space, so it stays a single argument.

### Paths Passed to a Shell

There are two places where `cmd2` knows that what is being completed will be passed to a shell: the
arguments of the [shell](./builtin_commands.md#shell) command, including its `!` shortcut, and
everything after a [pipe](./redirection.md). There `cmd2` also adds quotes when a match has a
character which is special to that shell, so the shell does not interpret it. For example,
completing `!ls my*` gives:

```text
(Cmd) !ls "my*file.txt"
```

On POSIX systems the characters which get a match quoted are a space, a quote character, `*`, `?`,
`[`, `{`, `}`, `(`, `)`, `<`, `>`, `|`, `&`, `;`, `$`, a backtick, a backslash, and a leading `#` or
`=`. Single quotes are used when a match has a `$`, a backtick, a backslash, or a double quote in
it, since a shell still interprets those within double quotes. On Windows the characters are the
ones `cmd.exe` documents as requiring quotes, other than `~`, and the quotes are always double
quotes.

There are limits to this:

- One pair of quotes goes around the whole argument, and it is chosen before you pick a match. So
when several matches are offered and only one of them needs quotes, they all get them. An
executable named `[` is common, so completing a shell command which could match it quotes every
match.
- For the same reason, one quote character is used for all the matches being offered. If any of them
has a single quote in it, double quotes are used, and a `$`, a backtick, or a backslash in any of
the matches is then still interpreted by a shell.
- A `%` in a match is still interpreted by `cmd.exe` on Windows.
- A [macro](./shortcuts_aliases_macros.md#macros) which runs a shell command is not covered. Its
arguments are completed as paths, but they are only quoted for a space. An alias for the `shell`
command is covered.
- Your own commands are not covered, even if they pass their arguments to a shell. Quote or escape
those arguments in the command.

## Included Tab Completion Functions

[cmd2.Cmd][] provides the following tab completion functions
Expand Down
14 changes: 13 additions & 1 deletion tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,12 @@
import os
import subprocess
import sys
from collections.abc import Callable
from collections.abc import (
Callable,
Iterable,
)
from contextlib import redirect_stderr
from pathlib import Path
from typing import (
TYPE_CHECKING,
ParamSpec,
Expand Down Expand Up @@ -61,6 +65,14 @@ def normalize(block: str) -> list[str]:
return [line.rstrip() for line in block.splitlines()]


def make_exes(directory: Path, names: Iterable[str]) -> None:
"""Create an executable file in directory for each name."""
for name in names:
exe = directory / name
exe.touch()
exe.chmod(0o755)


def run_cmd(app: cmd2.Cmd, cmd: str) -> tuple[list[str], list[str]]:
"""Run the command in app and return what it writes to app.stdout and sys.stderr."""

Expand Down
Loading
Loading