diff --git a/Documentation/config-yaml.rst b/Documentation/config-yaml.rst index 8383404..57f4f1f 100644 --- a/Documentation/config-yaml.rst +++ b/Documentation/config-yaml.rst @@ -381,6 +381,10 @@ Flash command can use special tags that are handled by NTFC: - ``$IMAGE_BIN`` is replaced by path to ``nuttx.bin``. - ``$IMAGE_HEX`` is replaced by path to ``nuttx.hex``. - ``$IMAGE_ELF`` is replaced by path to the core image (``elf_path``). +- ``$APPS_BINDIR`` is replaced by the application binaries directory + (kernel-mode builds). +- ``$APPS_IMG`` is replaced by the generated application filesystem + image (requires ``apps_image``). Example usage with ``st-flash`` tool: @@ -466,6 +470,11 @@ These fields are parsed by :class:`ntfc.coreconfig.CoreConfig`. - (Optional) Directory with kernel-mode application binaries. Defaults to the ``bin/`` directory next to the NuttX ELF for kernel-mode builds (``CONFIG_BUILD_KERNEL=y``) + * - ``apps_image`` + - (Optional) Generate a filesystem image with the application + binaries after build, e.g. ``apps_image: {type: romfs}``. The + image path is available as ``$APPS_IMG`` in the ``flash`` + command. Requires ``genromfs`` and a kernel-mode build * - ``defconfig`` - Path to NuttX defconfig (auto-build) * - ``elf_path`` diff --git a/Documentation/config.yaml b/Documentation/config.yaml index b091634..a0d6f90 100644 --- a/Documentation/config.yaml +++ b/Documentation/config.yaml @@ -77,6 +77,9 @@ product: # many products can be supported in tests (pro app_bindir: '' # (optional) directory with kernel-mode application binaries. # Defaults to the bin/ directory next to the NuttX ELF # for kernel-mode builds. + apps_image: # (optional) generate a filesystem image with the application + type: romfs # binaries after build (kernel-mode only, requires genromfs). + # The image path is available as $APPS_IMG in 'flash'. # NTFC can use pre-build image or build it from defconfig # the behavior will depend on the parameters specified in config. diff --git a/src/ntfc/builder.py b/src/ntfc/builder.py index 6e104f5..22d10b1 100644 --- a/src/ntfc/builder.py +++ b/src/ntfc/builder.py @@ -24,6 +24,7 @@ import re import shutil import subprocess +from functools import partial from pathlib import Path from typing import Any, Dict, List, Optional @@ -40,6 +41,9 @@ class NuttXBuilder: IMAGE_BIN_STR = "$IMAGE_BIN" IMAGE_HEX_STR = "$IMAGE_HEX" IMAGE_ELF_STR = "$IMAGE_ELF" + APPS_BINDIR_STR = "$APPS_BINDIR" + APPS_IMG_STR = "$APPS_IMG" + APPS_IMG_NAME = "apps.romfs.img" _KCONFIG_DISABLED_RE = re.compile( r"^#\s+(CONFIG_[A-Za-z0-9_]+)\s+is not set" ) @@ -348,7 +352,6 @@ def _run_build( "--build", str(build_path), ] - run_env = os.environ.copy() if env: run_env.update(env) # pragma: no cover @@ -428,8 +431,8 @@ def _build_core( if not already_build or self._rebuild: # pragma: no cover self._log_kconfig_overrides(kv_overrides) - # configure build - self._run_cmake( + configure = partial( + self._run_cmake, source=nuttx_dir, build=build_path, generator="Ninja", @@ -437,20 +440,28 @@ def _build_core( env=build_env, ) + # configure build + configure() + # apply Kconfig overrides to generated .config before build self._apply_kconfig_overrides( nuttx_conf_path, kv_overrides, cfg_cwd ) - # Regenerate include/nuttx/config.h after changing .config. - # Otherwise CMake can relink an image built with stale Kconfig - # values while the saved .config claims the override applied. - if kv_overrides: + # fill in defaults of suboptions the overrides + # unlocked (e.g. *_PROGNAME): applications are + # silently dropped at configure when these are + # missing from .config self._run_build_target( build_path, "olddefconfig", env=build_env ) + # application targets are registered at configure + # time from .config: configure again so overrides + # that enable new applications take effect + configure() + # build self._run_build(build_path, env=build_env) @@ -458,6 +469,79 @@ def _build_core( cores[core]["elf_path"] = nuttx_elf_path cores[core]["conf_path"] = nuttx_conf_path + if self._is_kernel_config(nuttx_conf_path): + # kernel-mode: applications are installed to /bin + # and hostfs mounts resolve relative to the process cwd + cores[core].setdefault( + "app_bindir", os.path.join(build_path, "bin") + ) + cores[core].setdefault("exec_cwd", build_path) + + self._make_apps_image(cores[core], build_path) + + def _make_apps_image( + self, core_cfg: Dict[str, Any], build_path: str + ) -> None: + """Generate a filesystem image with application binaries. + + Enabled with the per-core ``apps_image`` option; the image path + is registered as ``apps_img`` for the ``$APPS_IMG`` flash + placeholder. + """ + img_cfg = core_cfg.get("apps_image", None) + if not img_cfg: + return + + if not isinstance(img_cfg, dict): + raise BuilderConfigError("apps_image must be a mapping") + + img_type = img_cfg.get("type", "romfs") + if img_type != "romfs": + raise BuilderConfigError( + f"unsupported apps_image type: {img_type}" + ) + + bindir = core_cfg.get("app_bindir", None) + if not bindir: + raise BuilderConfigError( + "apps_image requires app_bindir (kernel-mode build)" + ) + + tool = shutil.which("genromfs") + if not tool: + raise BuilderConfigError("genromfs not found in PATH") + + img_path = os.path.join(build_path, self.APPS_IMG_NAME) + self._run_command([tool, "-f", img_path, "-d", bindir], env=None) + core_cfg["apps_img"] = img_path + + def _expand_flash_cmd( + self, flash_cmd: str, core_cfg: Dict[str, Any] + ) -> str: + """Expand image placeholders in a flash command.""" + parent = Path(core_cfg["elf_path"]).parent + values = { + self.IMAGE_BIN_STR: str(parent / "nuttx.bin"), + self.IMAGE_HEX_STR: str(parent / "nuttx.hex"), + self.IMAGE_ELF_STR: core_cfg["elf_path"], + self.APPS_BINDIR_STR: core_cfg.get("app_bindir", ""), + self.APPS_IMG_STR: core_cfg.get("apps_img", ""), + } + + for placeholder, value in values.items(): + flash_cmd = flash_cmd.replace(placeholder, value) + + return flash_cmd + + @staticmethod + def _is_kernel_config(conf_path: str) -> bool: + """Check if a generated .config selects a kernel build.""" + if not os.path.isfile(conf_path): + return False + + with open(conf_path, "r", encoding="utf-8") as f: + return any(line.strip() == "CONFIG_BUILD_KERNEL=y" for line in f) + def _reboot_core( self, core: str, cores: Dict[str, Any] ) -> None: # pragma: no cover @@ -474,13 +558,7 @@ def _flash_core( """Flash single core image.""" flash_cmd = cores[core].get("flash", None) if flash_cmd: - img_path = Path(cores[core]["elf_path"]) - image_hex = str(img_path.parent) + "/nuttx.hex" - image_bin = str(img_path.parent) + "/nuttx.bin" - - flash_cmd = flash_cmd.replace(self.IMAGE_BIN_STR, image_bin) - flash_cmd = flash_cmd.replace(self.IMAGE_HEX_STR, image_hex) - flash_cmd = flash_cmd.replace(self.IMAGE_ELF_STR, str(img_path)) + flash_cmd = self._expand_flash_cmd(flash_cmd, cores[core]) cmd = flash_cmd.split() diff --git a/tests/test_builder.py b/tests/test_builder.py index 1fb4ab8..cd5f7a0 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -79,6 +79,192 @@ def test_builder_init(): ) +def test_builder_kernel_mode_core_config(tmp_path) -> None: + config = copy.deepcopy(conf_dir) + config["config"]["build_dir"] = str(tmp_path) + config["product"]["cores"]["core0"]["defconfig"] = "dummy/path" + + build_path = tmp_path / "product-xxx-dummy" + build_path.mkdir() + (build_path / ".config").write_text("CONFIG_BUILD_KERNEL=y\n") + + b = NuttXBuilder(config) + b._run_command = builder_run_command_dummy + b._make_dir = builder_make_dir_dummy + b.build_all() + + core = b.new_conf()["product"]["cores"]["core0"] + assert core["app_bindir"] == str(build_path / "bin") + assert core["exec_cwd"] == str(build_path) + + +def test_builder_kernel_mode_keeps_user_overrides(tmp_path) -> None: + config = copy.deepcopy(conf_dir) + config["config"]["build_dir"] = str(tmp_path) + config["product"]["cores"]["core0"]["defconfig"] = "dummy/path" + config["product"]["cores"]["core0"]["app_bindir"] = "/custom/bin" + config["product"]["cores"]["core0"]["exec_cwd"] = "/custom/cwd" + + build_path = tmp_path / "product-xxx-dummy" + build_path.mkdir() + (build_path / ".config").write_text("CONFIG_BUILD_KERNEL=y\n") + + b = NuttXBuilder(config) + b._run_command = builder_run_command_dummy + b._make_dir = builder_make_dir_dummy + b.build_all() + + core = b.new_conf()["product"]["cores"]["core0"] + assert core["app_bindir"] == "/custom/bin" + assert core["exec_cwd"] == "/custom/cwd" + + +def test_builder_flat_mode_no_kernel_keys(tmp_path) -> None: + config = copy.deepcopy(conf_dir) + config["config"]["build_dir"] = str(tmp_path) + config["product"]["cores"]["core0"]["defconfig"] = "dummy/path" + + build_path = tmp_path / "product-xxx-dummy" + build_path.mkdir() + (build_path / ".config").write_text("CONFIG_BUILD_FLAT=y\n") + + b = NuttXBuilder(config) + b._run_command = builder_run_command_dummy + b._make_dir = builder_make_dir_dummy + b.build_all() + + core = b.new_conf()["product"]["cores"]["core0"] + assert "app_bindir" not in core + assert "exec_cwd" not in core + + # missing .config (dummy build) is treated as a flat build + assert ( + NuttXBuilder._is_kernel_config(str(tmp_path / "nonexistent")) is False + ) + + +def test_builder_reconfigures_after_kv_overrides(monkeypatch) -> None: + config = copy.deepcopy(conf_dir) + config["product"]["cores"]["core0"]["defconfig"] = "dummy/path" + config["product"]["cores"]["core0"]["kv"] = {"CONFIG_SYSTEM_X": "y"} + + calls = [] + + def run_command_capture(cmd, env): + calls.append(cmd) + + b = NuttXBuilder(config) + b._run_command = run_command_capture + b._make_dir = builder_make_dir_dummy + monkeypatch.setattr( + b, "_apply_kconfig_overrides", lambda *args, **kwargs: None + ) + + b.build_all() + + # application targets are registered at configure time: overrides + # require an olddefconfig (defaults of unlocked suboptions) and a + # second configure before the build + assert [cmd[0] for cmd in calls] == ["cmake"] * 4 + assert calls[1][-1] == "olddefconfig" + assert calls[0][:2] == calls[2][:2] + assert calls[3][:2] == ["cmake", "--build"] + assert "olddefconfig" not in calls[3] + + +def test_builder_expand_flash_cmd() -> None: + b = NuttXBuilder(copy.deepcopy(conf_dir)) + core_cfg = { + "elf_path": "bbb/core/nuttx", + "app_bindir": "bbb/core/bin", + "apps_img": "bbb/core/apps.romfs.img", + } + + cmd = b._expand_flash_cmd( + "flash $IMAGE_BIN $IMAGE_HEX $APPS_BINDIR $APPS_IMG", core_cfg + ) + assert cmd == ( + "flash bbb/core/nuttx.bin bbb/core/nuttx.hex " + "bbb/core/bin bbb/core/apps.romfs.img" + ) + + # placeholders without registered values expand to empty strings + cmd = b._expand_flash_cmd( + "flash $APPS_BINDIR $APPS_IMG", {"elf_path": "bbb/core/nuttx"} + ) + assert cmd == "flash " + + +def test_builder_apps_image(tmp_path, monkeypatch) -> None: + config = copy.deepcopy(conf_dir) + config["config"]["build_dir"] = str(tmp_path) + config["product"]["cores"]["core0"]["defconfig"] = "dummy/path" + config["product"]["cores"]["core0"]["apps_image"] = {"type": "romfs"} + + build_path = tmp_path / "product-xxx-dummy" + build_path.mkdir() + (build_path / ".config").write_text("CONFIG_BUILD_KERNEL=y\n") + + calls = [] + + def run_command_capture(cmd, env): + calls.append(cmd) + + monkeypatch.setattr( + "ntfc.builder.shutil.which", lambda tool: f"/usr/bin/{tool}" + ) + + b = NuttXBuilder(config) + b._run_command = run_command_capture + b._make_dir = builder_make_dir_dummy + b.build_all() + + core = b.new_conf()["product"]["cores"]["core0"] + img_path = str(build_path / "apps.romfs.img") + assert core["apps_img"] == img_path + assert calls[-1] == [ + "/usr/bin/genromfs", + "-f", + img_path, + "-d", + str(build_path / "bin"), + ] + + +def test_builder_apps_image_errors(tmp_path, monkeypatch) -> None: + def make_builder(config_txt, apps_image): + config = copy.deepcopy(conf_dir) + config["config"]["build_dir"] = str(tmp_path) + config["product"]["cores"]["core0"]["defconfig"] = "dummy/path" + config["product"]["cores"]["core0"]["apps_image"] = apps_image + + build_path = tmp_path / "product-xxx-dummy" + build_path.mkdir(exist_ok=True) + (build_path / ".config").write_text(config_txt) + + b = NuttXBuilder(config) + b._run_command = builder_run_command_dummy + b._make_dir = builder_make_dir_dummy + return b + + # unsupported image type + with pytest.raises(BuilderConfigError): + make_builder("CONFIG_BUILD_KERNEL=y\n", {"type": "vfat"}).build_all() + + # not a mapping + with pytest.raises(BuilderConfigError): + make_builder("CONFIG_BUILD_KERNEL=y\n", "romfs").build_all() + + # no application directory (flat build) + with pytest.raises(BuilderConfigError): + make_builder("CONFIG_BUILD_FLAT=y\n", {"type": "romfs"}).build_all() + + # genromfs not installed + monkeypatch.setattr("ntfc.builder.shutil.which", lambda tool: None) + with pytest.raises(BuilderConfigError): + make_builder("CONFIG_BUILD_KERNEL=y\n", {"type": "romfs"}).build_all() + + def test_builder_passes_build_env() -> None: config = copy.deepcopy(conf_dir) config["config"]["build_env"] = {"CC": "gcc-13", "CXX": "g++-13"} @@ -126,7 +312,8 @@ def test_builder_regenerates_config_after_kconfig_overrides() -> None: "--target", "olddefconfig", ] - assert calls[2] == ["cmake", "--build", "bbb/product-xxx-dummy"] + assert calls[0][:2] == calls[2][:2] + assert calls[3] == ["cmake", "--build", "bbb/product-xxx-dummy"] def test_builder_run_build_target_passes_env() -> None: @@ -498,11 +685,14 @@ def test_builder_applies_kv_before_build() -> None: def run_command_capture(cmd, env): calls.append(cmd) if "--build" not in cmd: + # cmake generates .config only when it does not exist yet expected_build_path.mkdir(parents=True, exist_ok=True) - expected_conf_path.write_text( - "# CONFIG_TEST_BOOL is not set\n" "CONFIG_TEST_STR=old\n", - encoding="utf-8", - ) + if not expected_conf_path.exists(): + expected_conf_path.write_text( + "# CONFIG_TEST_BOOL is not set\n" + "CONFIG_TEST_STR=old\n", + encoding="utf-8", + ) else: cfg_text = expected_conf_path.read_text(encoding="utf-8") assert "CONFIG_TEST_BOOL=y\n" in cfg_text @@ -522,11 +712,12 @@ def fake_apply_with_tool(conf_path, overrides, _cfg_cwd): with patch("ntfc.builder.logger.info", side_effect=logs.append): b.build_all() - assert len(calls) == 3 + # configure, olddefconfig and reconfigure after overrides, build + assert len(calls) == 4 assert calls[0][0] == "cmake" - assert calls[1][:2] == ["cmake", "--build"] - assert calls[1][-2:] == ["--target", "olddefconfig"] - assert calls[2][:2] == ["cmake", "--build"] + assert calls[1][-1] == "olddefconfig" + assert calls[2][0] == "cmake" + assert calls[3][:2] == ["cmake", "--build"] assert any( "Applying Kconfig overrides before build:" == msg for msg in logs )