From b582c758ded4b93c78594fa26bfa6eec72e72d05 Mon Sep 17 00:00:00 2001 From: Marco Casaroli Date: Sun, 26 Jul 2026 14:55:18 +0200 Subject: [PATCH 1/3] ntfc: Add optional line-buffered transport writes. Add the line_buffered per-core configuration option, disabled by default, to retain the existing byte-wise transport behavior. When enabled, the simulator and serial transports write a complete command in one operation. Buffered simulator writes also disable pexpect per-send delay. This improves local simulator performance and supports serial transports with reliable flow control. Document the option and cover both default and buffered command writes. Assisted-by: ChatGPT:GPT-5.6-Terra Signed-off-by: Marco Casaroli --- Documentation/config-yaml.rst | 24 ++++++++++++++++++ Documentation/config.yaml | 3 +++ src/ntfc/coreconfig.py | 5 ++++ src/ntfc/device/serial.py | 18 +++++++++----- src/ntfc/device/sim.py | 12 ++++++++- tests/device/test_serial.py | 26 +++++++++++++++++++ tests/device/test_sim.py | 47 +++++++++++++++++++++++++++++++++++ 7 files changed, 128 insertions(+), 7 deletions(-) diff --git a/Documentation/config-yaml.rst b/Documentation/config-yaml.rst index 3b66322..e23ad76 100644 --- a/Documentation/config-yaml.rst +++ b/Documentation/config-yaml.rst @@ -118,6 +118,30 @@ checks. flash_only: true # core1 is built/flashed only +Line-buffered command writes +============================ + +By default, NTFC sends a command one byte at a time. This is the safest mode +for serial targets without flow control and remains the default. + +Set ``line_buffered: true`` for a core to write a complete command in one +transport operation instead. This reduces host-side overhead for ``sim`` and +can also be enabled for serial targets with reliable flow control. + +.. code-block:: yaml + + product: + name: "product-name" + cores: + core0: + name: "main" + device: "sim" + line_buffered: true + +For the simulator, line-buffered mode also disables the pexpect per-send +delay. Do not enable this option for serial targets that cannot +reliably accept a full command at once. + **SMP (Symmetric Multi-Processing)** In SMP mode, all cores share the same device instance, coordinated by diff --git a/Documentation/config.yaml b/Documentation/config.yaml index 1b38027..6d17628 100644 --- a/Documentation/config.yaml +++ b/Documentation/config.yaml @@ -88,6 +88,9 @@ product: # many products can be supported in tests (pro poweroff: '' # (optional) System command to power off DUT flash_only: false # (optional) build/flash this core but skip boot checks, # requirement validation, logs, and test execution on it + line_buffered: false # (optional) send each command in one transport write. + # Keep false for the default byte-wise behavior. + # Enable for sim or a serial transport with flow control. dcmake: # (optional) Defines passed to CMake build DEFINE1: "VALUE1" DEFINE2: "VALUE2" diff --git a/src/ntfc/coreconfig.py b/src/ntfc/coreconfig.py index 263725a..7026170 100644 --- a/src/ntfc/coreconfig.py +++ b/src/ntfc/coreconfig.py @@ -193,6 +193,11 @@ def app_bindir(self) -> Optional[str]: return os.path.join(os.path.dirname(self.elf_path), "bin") return None + @property + def line_buffered(self) -> bool: + """Return whether commands are sent in one transport write.""" + return bool(self._config.get("line_buffered", False)) + def kv_check(self, cfg: str) -> Any: """Check Kconfig option and return its value. diff --git a/src/ntfc/device/serial.py b/src/ntfc/device/serial.py index fca70fb..808bb98 100644 --- a/src/ntfc/device/serial.py +++ b/src/ntfc/device/serial.py @@ -101,13 +101,19 @@ def _write(self, data: bytes) -> None: assert self._ser - # send char by char to avoid line length full - for c in data: - self._ser.write(bytes([c])) + if self._conf.line_buffered: + if data[-1] != ord("\n"): + data += b"\n" - # add new line if missing - if data[-1] != ord("\n"): - self._ser.write(b"\n") # pragma: no cover + self._ser.write(data) + else: + # send char by char to avoid line length full + for c in data: + self._ser.write(bytes([c])) + + # add new line if missing + if data[-1] != ord("\n"): + self._ser.write(b"\n") # pragma: no cover # read all garbage left by character echo _ = self._read_all(timeout=0) diff --git a/src/ntfc/device/sim.py b/src/ntfc/device/sim.py index 720c5fc..6a19eaa 100644 --- a/src/ntfc/device/sim.py +++ b/src/ntfc/device/sim.py @@ -49,7 +49,9 @@ def _start_impl(self) -> None: uptime = self._conf.uptime # open host-based emulation - self.host_open(cmd, uptime) + child = self.host_open(cmd, uptime) + if self._conf.line_buffered: + child.delaybeforesend = 0 @property def name(self) -> str: @@ -63,6 +65,14 @@ def _write(self, data: bytes) -> None: assert self._child + if self._conf.line_buffered: + if data[-1] != ord("\n"): + # Sometimes sim misses a single trailing newline. + data += b"\n\n" + + self._child.send(data) + return + # send char by char to avoid line length full for c in data: self._child.send(bytes([c])) diff --git a/tests/device/test_serial.py b/tests/device/test_serial.py index 74822a2..b08c56d 100644 --- a/tests/device/test_serial.py +++ b/tests/device/test_serial.py @@ -93,6 +93,32 @@ def test_device_sim_internals(serial_config, serial_pair): assert ser._write(b"a") is None +def test_device_serial_line_buffered_write(serial_config): + ser = DeviceSerial(serial_config) + sent = [] + + class FakeSerial: + def write(self, data): + sent.append(data) + + def read(self, size): + return b"" + + ser._ser = FakeSerial() + + assert ser._write(b"abc") is None + assert sent == [b"a", b"b", b"c", b"\n"] + + sent.clear() + serial_config._config["line_buffered"] = True + assert ser._write(b"abc") is None + assert sent == [b"abc\n"] + + sent.clear() + assert ser._write(b"abc\n") is None + assert sent == [b"abc\n"] + + def test_device_sim_init(serial_config, serial_pair): ser = DeviceSerial(serial_config) diff --git a/tests/device/test_sim.py b/tests/device/test_sim.py index f9523c3..b7c5293 100644 --- a/tests/device/test_sim.py +++ b/tests/device/test_sim.py @@ -47,6 +47,7 @@ def test_device_sim_start_opens_host(): config.os = "nuttx" config.read_poll_interval = 0.1 config.elf_path = "/tmp/nuttx-sim" + config.line_buffered = False config.uptime = 3 sim = DeviceSim(config) @@ -64,12 +65,31 @@ def fake_host_open(cmd, uptime): assert called["uptime"] == 3 +def test_device_sim_line_buffered_start_disables_send_delay(): + with patch("ntfc.coreconfig.CoreConfig") as mockdevice: + config = mockdevice.return_value + config.elf_path = "/tmp/nuttx-sim" + config.line_buffered = True + config.uptime = 0 + sim = DeviceSim(config) + + class FakeChild: + delaybeforesend = 0.05 + + child = FakeChild() + sim.host_open = lambda *_args: child + + sim.start() + assert child.delaybeforesend == 0 + + def test_device_sim_write_adds_newline(): with patch("ntfc.coreconfig.CoreConfig") as mockdevice: config = mockdevice.return_value config.os = "nuttx" config.read_poll_interval = 0.1 config.elf_path = "/tmp/nuttx-sim" + config.line_buffered = False sim = DeviceSim(config) sent = [] @@ -94,6 +114,7 @@ def test_device_sim_write_no_extra_newline(): config.os = "nuttx" config.read_poll_interval = 0.1 config.elf_path = "/tmp/nuttx-sim" + config.line_buffered = False sim = DeviceSim(config) sent = [] @@ -109,3 +130,29 @@ def send(self, data): sim._write(b"abc\n") assert sent == [b"a", b"b", b"c", b"\n"] + + +def test_device_sim_line_buffered_write(): + with patch("ntfc.coreconfig.CoreConfig") as mockdevice: + config = mockdevice.return_value + config.elf_path = "/tmp/nuttx-sim" + config.line_buffered = True + sim = DeviceSim(config) + + sent = [] + + class FakeChild: + def isalive(self): + return True + + def send(self, data): + sent.append(data) + + sim._child = FakeChild() + + sim._write(b"abc") + assert sent == [b"abc\n\n"] + + sent.clear() + sim._write(b"abc\n") + assert sent == [b"abc\n"] From 6ed87c8620e7ebd061bb10f214b5fbe2d5610eee Mon Sep 17 00:00:00 2001 From: Marco Casaroli Date: Mon, 31 Aug 2026 09:54:12 +0200 Subject: [PATCH 2/3] ntfc: Set the line-buffered send delay where a reopen goes through. The per-send delay was turned off by DeviceSim._start_impl() after host_open() returned. A device that crashes comes back through _dev_reopen(), which calls host_open() directly and never reaches the device's own start, so the setting was lost for the rest of the run. Setting it in host_open() keeps it across a reopen, and covers QEMU as well as the simulator, since both are host-based devices. It changes nothing for either unless line_buffered is set, which is off by default. The test moves with the behaviour: it now opens a device, reopens it and checks the delay is still off. Signed-off-by: Marco Casaroli --- src/ntfc/device/host.py | 11 ++++++++++- src/ntfc/device/sim.py | 4 +--- tests/device/test_host.py | 20 ++++++++++++++++++++ tests/device/test_sim.py | 18 ------------------ 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/ntfc/device/host.py b/src/ntfc/device/host.py index fb11750..6ccba99 100644 --- a/src/ntfc/device/host.py +++ b/src/ntfc/device/host.py @@ -133,10 +133,19 @@ def host_open(self, cmd: List[str], uptime: int = 0) -> pexpect.spawn: self._cmd = cmd logger.info(f"spawn cmd: {''.join(cmd)}") - self._child = pexpect.spawn( + child = pexpect.spawn( "".join(cmd), timeout=10, maxread=20000, cwd=self._cwd ) + if self._conf.line_buffered: + # A line-buffered transport writes a whole command at once, so the + # per-send delay pexpect inserts between characters buys nothing. + # This is set here rather than by the caller so that it survives + # _dev_reopen(), which comes back through this function. + child.delaybeforesend = 0 + + self._child = child + time.sleep(uptime) ret = self._wait_for_boot(self._conf.boot_timeout) diff --git a/src/ntfc/device/sim.py b/src/ntfc/device/sim.py index 6a19eaa..57cb226 100644 --- a/src/ntfc/device/sim.py +++ b/src/ntfc/device/sim.py @@ -49,9 +49,7 @@ def _start_impl(self) -> None: uptime = self._conf.uptime # open host-based emulation - child = self.host_open(cmd, uptime) - if self._conf.line_buffered: - child.delaybeforesend = 0 + self.host_open(cmd, uptime) @property def name(self) -> str: diff --git a/tests/device/test_host.py b/tests/device/test_host.py index 710334a..1d1f57c 100644 --- a/tests/device/test_host.py +++ b/tests/device/test_host.py @@ -48,6 +48,26 @@ def _write(self, data): self._child.send(b"\n\n") +def test_device_host_line_buffered_survives_reopen(envconfig_dummy): + """A line-buffered device keeps its send delay off across a reopen.""" + + conf = envconfig_dummy.product[0].cfg_core(0) + conf._config["line_buffered"] = True + + path = "./tests/resources/nuttx/sim/nuttx" + dev = DeviceHost2(conf) + + child = dev.host_open([path]) + assert child.delaybeforesend == 0 + + # a crash brings the device back through host_open() rather than through + # the device's own start, so the setting has to be made there + child = dev._dev_reopen() + assert child.delaybeforesend == 0 + + dev.stop() + + def test_device_host_open(envconfig_dummy, monkeypatch): conf = envconfig_dummy.product[0].cfg_core(0) diff --git a/tests/device/test_sim.py b/tests/device/test_sim.py index b7c5293..7cbfc2c 100644 --- a/tests/device/test_sim.py +++ b/tests/device/test_sim.py @@ -65,24 +65,6 @@ def fake_host_open(cmd, uptime): assert called["uptime"] == 3 -def test_device_sim_line_buffered_start_disables_send_delay(): - with patch("ntfc.coreconfig.CoreConfig") as mockdevice: - config = mockdevice.return_value - config.elf_path = "/tmp/nuttx-sim" - config.line_buffered = True - config.uptime = 0 - sim = DeviceSim(config) - - class FakeChild: - delaybeforesend = 0.05 - - child = FakeChild() - sim.host_open = lambda *_args: child - - sim.start() - assert child.delaybeforesend == 0 - - def test_device_sim_write_adds_newline(): with patch("ntfc.coreconfig.CoreConfig") as mockdevice: config = mockdevice.return_value From 33c92fd2b9a8f2f7a672c74799c23a87b5e24464 Mon Sep 17 00:00:00 2001 From: Marco Casaroli Date: Mon, 31 Aug 2026 10:59:24 +0200 Subject: [PATCH 3/3] ntfc: Give the line-buffered sim test an OS to construct with. DeviceSim goes through get_os(), which refuses a configuration whose os it does not recognise. The other sim tests set it on the mock and this one did not, so it raised before reaching what it meant to check. Signed-off-by: Marco Casaroli --- tests/device/test_sim.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/device/test_sim.py b/tests/device/test_sim.py index 7cbfc2c..b4397ae 100644 --- a/tests/device/test_sim.py +++ b/tests/device/test_sim.py @@ -119,6 +119,7 @@ def test_device_sim_line_buffered_write(): config = mockdevice.return_value config.elf_path = "/tmp/nuttx-sim" config.line_buffered = True + config.os = "nuttx" sim = DeviceSim(config) sent = []