diff --git a/Documentation/writing-test-cases.rst b/Documentation/writing-test-cases.rst index 7637292..1f3a30e 100644 --- a/Documentation/writing-test-cases.rst +++ b/Documentation/writing-test-cases.rst @@ -192,7 +192,11 @@ Execute NSH command and verify output: Decorators: -- ``@pytest.mark.cmd_check("symbol_name")``: Verify ELF symbol exists +- ``@pytest.mark.cmd_check("symbol_name")``: Verify ELF symbol exists. + On kernel-mode targets (``CONFIG_BUILD_KERNEL=y``) the marker is first + matched against application file names (a trailing ``_main`` maps to + the file name, so ``hello_main`` matches the ``hello`` binary) and + then against symbols in the unstripped application binaries - ``@pytest.mark.dep_config("CONFIG_X", "CONFIG_Y")``: Skip if configs not enabled diff --git a/src/ntfc/core.py b/src/ntfc/core.py index 239b541..9de18fc 100644 --- a/src/ntfc/core.py +++ b/src/ntfc/core.py @@ -35,6 +35,7 @@ from ntfc.command_builder import CommandBuilder from ntfc.coreconfig import CoreConfig from ntfc.device.common import CmdReturn, CmdStatus +from ntfc.lib.elf.app_bindir import match_command, symbol_patterns from ntfc.log.logger import logger if TYPE_CHECKING: @@ -72,6 +73,8 @@ class CoreStatus(_Enum): class ProductCore: """This class implements product core under test.""" + _PATH_NAME_RE = re.compile(r"[A-Za-z0-9_.+-]+") + def __init__( self, device: "DeviceCommon", @@ -99,6 +102,7 @@ def __init__( list(ignored_cores) if ignored_cores is not None else ["dsp"] ) self._builder = CommandBuilder(device.prompt, device.no_cmd) + self._runtime_cmds: Optional[List[str]] = None self._prompt = device.prompt self._main_prompt = self._prompt @@ -527,67 +531,82 @@ def start(self) -> None: """Start device.""" self._device.start() + def _check_cmd_runtime(self, cmd_pattern: str) -> bool: + """Check a command against the target PATH listing. + + The listing is fetched once from the running target and cached; + a failed listing is not cached so it is retried on next use. + """ + if self._runtime_cmds is None: + result = self.sendCommandReadUntilPattern( + f"ls {self._conf.path_initial}", timeout=5 + ) + if result.status != CmdStatus.SUCCESS: + return False + + output_lower = result.output.lower() + no_cmd = str(self._device.no_cmd).lower() + if no_cmd in output_lower or any( + line.lstrip().lower().startswith("ls:") + for line in result.output.splitlines() + ): + return False + + # drop the command echo line; path headers and the prompt + # do not match the name pattern + tokens = " ".join(result.output.splitlines()[1:]).split() + self._runtime_cmds = [ + token + for token in tokens + if self._PATH_NAME_RE.fullmatch(token) + ] + + return match_command(cmd_pattern, self._runtime_cmds) + def check_cmd(self, cmd_pattern: str) -> bool: - """Check if a command pattern is available in the ELF binary. + """Check if a command pattern is available on the core. - This method validates whether a specific command or set of - commands is present in the core's ELF binary by searching for - corresponding symbols. It supports both single commands and - alternative command patterns separated by '|'. + Commands are resolved from the application binaries, the running + target, the device ELF parser or the shell help output, in that + order. - :param cmd_pattern: Command pattern to check for. Can contain - alternatives separated by '|' - (e.g., 'test1|test2') + :param cmd_pattern: Command pattern, may contain alternatives + separated by '|' (e.g. 'test1|test2') :return: True if the command pattern is found, False otherwise - - Note: - - Requires ELF parser to be available in device - - Supports alternative patterns with '|' - - Used to validate core capabilities before executing """ - # Check if device supports command checking + # Kernel-mode: resolve against the application binary directory + # so runtime checks agree with collection-time filtering + if self._conf.has_app_bindir: + return self._conf.cmd_check(cmd_pattern) + + if self._conf.is_kernel_build: + # prebuilt image without host binaries: discover the + # command set once from the running target + return self._check_cmd_runtime(cmd_pattern) + + # Devices that expose their own ELF parser resolve the command + # from its symbols if hasattr(self._device, "elf_parser") and self._device.elf_parser: logger.debug(f"Checking command pattern: {cmd_pattern}") - # Split by '|' to support alternative patterns - alternatives = ( - cmd_pattern.split("|") if "|" in cmd_pattern else [cmd_pattern] - ) - - for pattern in alternatives: - # For cmocka tests, append _main to symbol name - symbol_pattern_str: str = ( - f"{pattern}_main" if "cmocka" in pattern else pattern - ) - - # Support regex wildcards - if ".*" in symbol_pattern_str: - symbol_pattern = re.compile(symbol_pattern_str) - else: - tmp = symbol_pattern_str - symbol_pattern = tmp # type: ignore[assignment] - - if self._device.elf_parser.has_symbol(symbol_pattern): + for symbol in symbol_patterns(cmd_pattern): + if self._device.elf_parser.has_symbol(symbol): return True return False - # Fallback: try to execute the command and check if it exists + # Fallback: check the command against the shell help output logger.warning( "ELF parser not available, trying command check for: " f"{cmd_pattern}" ) - # Send 'help' command to check if command exists result = self.sendCommandReadUntilPattern("help", timeout=5) if result.status == CmdStatus.SUCCESS: - # Check if any of the command alternatives are in the help output - alternatives = ( - cmd_pattern.split("|") if "|" in cmd_pattern else [cmd_pattern] + return any( + pattern.lower() in result.output.lower() + for pattern in cmd_pattern.split("|") ) - for pattern in alternatives: - if pattern.lower() in result.output.lower(): - return True return False diff --git a/src/ntfc/coreconfig.py b/src/ntfc/coreconfig.py index 7026170..ed06769 100644 --- a/src/ntfc/coreconfig.py +++ b/src/ntfc/coreconfig.py @@ -21,8 +21,10 @@ """Product core configuration handler.""" import os +from functools import cached_property from typing import Any, Dict, Optional, Union +from ntfc.lib.elf.app_bindir import AppBinDir, symbol_patterns from ntfc.lib.elf.elf_parser import ElfParser @@ -211,10 +213,41 @@ def kv_check(self, cfg: str) -> Any: return self._kv_values.get(cfg, False) + @cached_property + def _appbin(self) -> Optional[AppBinDir]: + """Return the installed kernel-mode application binaries.""" + bindir = self.app_bindir + if not bindir or not os.path.isdir(bindir): + return None + + return AppBinDir(bindir) + + @property + def has_app_bindir(self) -> bool: + """Return True when kernel-mode application binaries are found.""" + return self._appbin is not None + + @property + def path_initial(self) -> str: + """Return the initial target PATH, see CONFIG_PATH_INITIAL.""" + return str(self.kv_check("CONFIG_PATH_INITIAL") or "/system/bin") + def cmd_check(self, cmd: str, core: int = 0) -> bool: - """Check if command is available in binary.""" + """Check if command is available in binary. + + Kernel-mode cores resolve commands against the application binary + directory; otherwise the symbol must exist in the core ELF. + """ + if self._appbin: + if self._appbin.has_command(cmd): + return True + # not an application: NSH builtins and test entry points + # are symbols inside the application binaries + return self._appbin.has_symbol(cmd) + if not self._elf: raise AttributeError("no elf data") - symbol_name = f"{cmd}_main" if "cmocka" in cmd else cmd - return self._elf.has_symbol(symbol_name) + return any( + self._elf.has_symbol(symbol) for symbol in symbol_patterns(cmd) + ) diff --git a/src/ntfc/device/host.py b/src/ntfc/device/host.py index d632387..e188e65 100644 --- a/src/ntfc/device/host.py +++ b/src/ntfc/device/host.py @@ -61,6 +61,18 @@ def pid(self) -> Optional[int]: """Return the spawned child PID, or ``None`` before start.""" return self._child.pid if self._child else None + def _image_path(self) -> str: + """Return the image path, absolute when spawning in another cwd. + + Relative paths are relative to the NTFC working directory, not + to the spawn directory. + """ + elf = self._conf.elf_path + if not elf: + raise IOError + + return os.path.abspath(elf) if self._cwd else str(elf) + def _dev_is_health_priv(self) -> bool: """Check if the host device is OK.""" if not self._child: diff --git a/src/ntfc/device/qemu.py b/src/ntfc/device/qemu.py index a655d26..a565003 100644 --- a/src/ntfc/device/qemu.py +++ b/src/ntfc/device/qemu.py @@ -44,12 +44,9 @@ def __init__(self, conf: "CoreConfig"): def _start_impl(self) -> None: """Start QEMU emulator implementation.""" - elf = self._conf.elf_path + elf = self._image_path() exec_path = self._conf.exec_path exec_args = self._conf.exec_args - - if not elf: - raise IOError if not exec_path: raise KeyError("no exec_path in configuration file!") diff --git a/src/ntfc/device/sim.py b/src/ntfc/device/sim.py index d4ebeb5..8da475f 100644 --- a/src/ntfc/device/sim.py +++ b/src/ntfc/device/sim.py @@ -44,11 +44,7 @@ def __init__(self, conf: "CoreConfig"): def _start_impl(self) -> None: """Start sim emulator implementation.""" - elf = self._conf.elf_path - if not elf: - raise IOError - - cmd = [elf] + cmd = [self._image_path()] uptime = self._conf.uptime # open host-based emulation diff --git a/src/ntfc/testfilter.py b/src/ntfc/testfilter.py index 155c882..00d5bab 100644 --- a/src/ntfc/testfilter.py +++ b/src/ntfc/testfilter.py @@ -85,12 +85,12 @@ def check_test_support( reason = f"Required config '{d}' not enabled" break - # command available in ELF + # command available on the target if skip is False: for c in cmd: if self._config.cmd_check(c) is False: skip = True - reason = f"Required symbol '{c}' not found in ELF" + reason = f"Required command '{c}' not available" break # check extra parameters diff --git a/tests/device/test_qemu.py b/tests/device/test_qemu.py index c99741e..008fd91 100644 --- a/tests/device/test_qemu.py +++ b/tests/device/test_qemu.py @@ -18,6 +18,7 @@ # ############################################################################ +import os from unittest.mock import patch import pytest @@ -89,6 +90,7 @@ def test_device_qemu_open(): config.exec_path = "" config.exec_args = "" config.elf_path = "" + config.exec_cwd = None qemu = DeviceQemu(config) @@ -175,3 +177,27 @@ def host_open_dummy4(cmd, uptime): config.uptime = 3 qemu.start() + + +def test_device_qemu_exec_cwd_absolute_image(tmp_path): + + from ntfc.coreconfig import CoreConfig + + config = CoreConfig( + { + "name": "t", + "device": "qemu", + "exec_path": "qemu-system-riscv64", + "exec_args": "-nographic", + "exec_cwd": str(tmp_path), + } + ) + # relative image path, resolved against the NTFC cwd at spawn time + config._config["elf_path"] = "some/image" + + qemu = DeviceQemu(config) + cmds = [] + qemu.host_open = lambda cmd, uptime: cmds.append(cmd) + qemu.start() + + assert cmds[0][2] == "-kernel " + os.path.abspath("some/image") diff --git a/tests/device/test_sim.py b/tests/device/test_sim.py index b4397ae..0d76ce6 100644 --- a/tests/device/test_sim.py +++ b/tests/device/test_sim.py @@ -139,3 +139,33 @@ def send(self, data): sent.clear() sim._write(b"abc\n") assert sent == [b"abc\n"] + + +def test_device_sim_exec_cwd_absolute_image(tmp_path): + + import os + + from ntfc.coreconfig import CoreConfig + + config = CoreConfig( + {"name": "t", "device": "sim", "exec_cwd": str(tmp_path)} + ) + # relative image path, resolved against the NTFC cwd at spawn time + config._config["elf_path"] = "some/image" + + sim = DeviceSim(config) + cmds = [] + sim.host_open = lambda cmd, uptime: cmds.append(cmd) + sim.start() + + assert cmds[0] == [os.path.abspath("some/image")] + + # without exec_cwd the image path is passed through unchanged + config = CoreConfig({"name": "t", "device": "sim"}) + config._config["elf_path"] = "some/image" + + sim = DeviceSim(config) + sim.host_open = lambda cmd, uptime: cmds.append(cmd) + sim.start() + + assert cmds[1] == ["some/image"] diff --git a/tests/test_core.py b/tests/test_core.py index 838513d..7439831 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -585,8 +585,6 @@ def test_core_check_cmd_with_elf_parser(envconfig_dummy): # Test regex wildcard pattern mock_elf_parser.has_symbol.side_effect = None mock_elf_parser.has_symbol.return_value = True - import re - assert p.check_cmd("test.*") is True # The pattern should be compiled as regex call_args = mock_elf_parser.has_symbol.call_args @@ -595,3 +593,115 @@ def test_core_check_cmd_with_elf_parser(envconfig_dummy): # Test all alternatives not found mock_elf_parser.has_symbol.return_value = False assert p.check_cmd("nonexistent1|nonexistent2") is False + + # Test with failed help command + dev.send_cmd_read_until_pattern.return_value = CmdReturn( + CmdStatus.TIMEOUT + ) + assert p.check_cmd("test") is False + + +def test_core_check_cmd_kernel_mode(tmp_path): + """Test check_cmd resolves via app_bindir on kernel builds.""" + from ntfc.coreconfig import CoreConfig + + kernel_cfg = tmp_path / "kv_config" + kernel_cfg.write_text("CONFIG_BUILD_KERNEL=y\n") + bindir = tmp_path / "bin" + bindir.mkdir() + (bindir / "hello").write_bytes(b"\x7fELF" + b"\x00" * 12) + + conf = CoreConfig( + { + "name": "t", + "conf_path": str(kernel_cfg), + "app_bindir": str(bindir), + } + ) + + with patch("ntfc.device.common.DeviceCommon") as mockdevice: + p = ProductCore(mockdevice.return_value, conf) + + # resolved from the application directory, device is not queried + assert p.check_cmd("hello") is True + assert p.check_cmd("hello_main") is True + assert p.check_cmd("nonexistent") is False + mockdevice.return_value.send_cmd_read_until_pattern.assert_not_called() + + +def test_core_check_cmd_kernel_runtime_fallback(tmp_path, monkeypatch): + """Test check_cmd discovers commands from the running target.""" + from ntfc.coreconfig import CoreConfig + + kernel_cfg = tmp_path / "kv_config" + kernel_cfg.write_text( + 'CONFIG_BUILD_KERNEL=y\nCONFIG_PATH_INITIAL="/system/bin"\n' + ) + # a kernel core whose application binaries are not on the host: + # the command set has to come from the running target + conf = CoreConfig( + { + "name": "t", + "conf_path": str(kernel_cfg), + "elf_path": "./tests/resources/nuttx/sim/nuttx", + } + ) + assert conf.has_app_bindir is False + + with patch("ntfc.device.common.DeviceCommon") as mockdevice: + p = ProductCore(mockdevice.return_value, conf) + + calls = [] + status = [CmdStatus.TIMEOUT] + + def fake_send(cmd, pattern=None, args=None, timeout=30): + calls.append(cmd) + output = "ls /system/bin\n/system/bin:\n hello\n init\n sh\nnsh> " + return CmdReturn(status[0], output=output) + + monkeypatch.setattr(p, "sendCommandReadUntilPattern", fake_send) + + # listing failure is not cached + assert p.check_cmd("hello") is False + assert len(calls) == 1 + + status[0] = CmdStatus.SUCCESS + assert p.check_cmd("hello") is True + assert p.check_cmd("hello_main") is True + assert p.check_cmd("missing|sh") is True + assert p.check_cmd("missing") is False + + # target was listed once after the failed attempt + assert calls == ["ls /system/bin", "ls /system/bin"] + + +def test_core_check_cmd_kernel_runtime_does_not_cache_ls_error( + tmp_path, monkeypatch +): + """Do not interpret words from an ls error as target commands.""" + from ntfc.coreconfig import CoreConfig + + kernel_cfg = tmp_path / "kv_config" + kernel_cfg.write_text("CONFIG_BUILD_KERNEL=y\n") + conf = CoreConfig({"name": "t", "conf_path": str(kernel_cfg)}) + + with patch("ntfc.device.common.DeviceCommon") as mockdevice: + p = ProductCore(mockdevice.return_value, conf) + calls = [] + + def fake_send(cmd, pattern=None, args=None, timeout=30): + calls.append(cmd) + return CmdReturn( + CmdStatus.SUCCESS, + output=( + "ls /system/bin\n" + "ls: /system/bin: No such file or directory\n" + "nsh> " + ), + ) + + monkeypatch.setattr(p, "sendCommandReadUntilPattern", fake_send) + + assert p.check_cmd("No") is False + assert p.check_cmd("file") is False + assert calls == ["ls /system/bin", "ls /system/bin"] diff --git a/tests/test_coreconfig.py b/tests/test_coreconfig.py index 082d3f0..9eb666e 100644 --- a/tests/test_coreconfig.py +++ b/tests/test_coreconfig.py @@ -18,6 +18,8 @@ # ############################################################################ +import shutil + import pytest from ntfc.coreconfig import CoreConfig @@ -164,6 +166,48 @@ def test_core_config_app_bindir(tmp_path): } assert CoreConfig(conf).app_bindir is None + # a derived directory that does not exist is not a command source + conf = { + "name": "t", + "conf_path": str(kernel_cfg), + "elf_path": "./tests/resources/nuttx/sim/nuttx", + } + core_conf = CoreConfig(conf) + assert core_conf.app_bindir == "./tests/resources/nuttx/sim/bin" + assert core_conf.has_app_bindir is False + + +def test_core_config_cmd_check_kernel_mode(tmp_path): + kernel_cfg = tmp_path / "kv_config" + kernel_cfg.write_text("CONFIG_BUILD_KERNEL=y\n") + bindir = tmp_path / "bin" + bindir.mkdir() + (bindir / "hello").write_bytes(b"\x7fELF" + b"\x00" * 12) + + conf = { + "name": "t", + "conf_path": str(kernel_cfg), + "app_bindir": str(bindir), + } + + p = CoreConfig(conf) + assert p.has_app_bindir is True + # command resolution by application file name, no ELF configured + assert p.cmd_check("hello") is True + assert p.cmd_check("hello_main") is True + assert p.cmd_check("cmd_df") is False + + # symbol fallback over unstripped binaries in bin_debug + debug = tmp_path / "bin_debug" + debug.mkdir() + shutil.copy("./tests/resources/nuttx/sim/nuttx", debug / "sh") + + p = CoreConfig(conf) + assert p.cmd_check("cmd_df") is True + assert p.cmd_check("missing|cmd_df") is True + assert p.cmd_check("cmd_d.*") is True + assert p.cmd_check("no_such_symbol_xyz") is False + def test_core_config_read_poll_interval() -> None: assert CoreConfig({"name": "test"}).read_poll_interval == 0.1 diff --git a/tests/test_filtertest.py b/tests/test_filtertest.py index e7df74b..cf85ce6 100644 --- a/tests/test_filtertest.py +++ b/tests/test_filtertest.py @@ -57,7 +57,7 @@ def test_filterest_filter(): f.extract_test_requirements = mock_extract_test_requirements2 skip, reason = f.check_test_support(None) assert skip is True - assert reason == "Required symbol 'CMD1' not found in ELF" + assert reason == "Required command 'CMD1' not available" config.kv_check.return_value = True config.cmd_check.return_value = True