From edac81657a5eb4a8b8ea8045d50221746c5aca50 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 11:04:13 -0700 Subject: [PATCH 01/23] models: Describe pool memory as kernel chunks plus per-node requests The kernel now owns pool memory as a list of contiguous chunks, each on one NUMA node, and accepts per-node size requests instead of a single donated range, so MemoryAllocation can no longer be a base and a size. Model it as a list of PoolMemoryRegion (base, size, node) read back from the kernel plus a node-to-bytes request map, keep memory_pool_base, memory_pool_bytes and memory_pool_end as derived properties so the existing consumers keep working, and convert every constructor site in the tree and the tests. Signed-off-by: Cong Wang --- src/kerf/dtc/parser.py | 9 +++------ src/kerf/init/main.py | 4 ++-- src/kerf/models.py | 33 +++++++++++++++++++++++++++++---- tests/conftest.py | 4 ++-- tests/create_test_dtb.py | 4 ++-- tests/demo.py | 4 ++-- tests/test_baseline.py | 5 ++--- tests/test_kerf.py | 4 ++-- tests/test_models.py | 30 ++++++++++++++++++++++++++---- 9 files changed, 70 insertions(+), 27 deletions(-) diff --git a/src/kerf/dtc/parser.py b/src/kerf/dtc/parser.py index 0184739..e484291 100644 --- a/src/kerf/dtc/parser.py +++ b/src/kerf/dtc/parser.py @@ -33,6 +33,7 @@ MemoryAllocation, NUMANode, OverlayInstanceData, + PoolMemoryRegion, TopologySection, ) @@ -138,8 +139,6 @@ def _build_global_tree(self) -> GlobalDeviceTree: memory=MemoryAllocation( total_bytes=0, host_reserved_bytes=0, - memory_pool_base=0, - memory_pool_bytes=0 ), topology=None, devices={} @@ -246,8 +245,7 @@ def _parse_memory_allocation(self, resources_node: int) -> MemoryAllocation: return MemoryAllocation( total_bytes=total_bytes, host_reserved_bytes=host_reserved_bytes, - memory_pool_base=memory_pool_base, - memory_pool_bytes=memory_pool_bytes + regions=[PoolMemoryRegion(base=memory_pool_base, size=memory_pool_bytes)] ) def _parse_devices(self, resources_node: int) -> Dict[str, DeviceInfo]: @@ -798,8 +796,7 @@ def _parse_memory_from_dts(self, dts_content: str) -> MemoryAllocation: return MemoryAllocation( total_bytes=total_bytes, host_reserved_bytes=host_reserved_bytes, - memory_pool_base=memory_pool_base, - memory_pool_bytes=memory_pool_bytes + regions=[PoolMemoryRegion(base=memory_pool_base, size=memory_pool_bytes)] ) def _parse_devices_from_dts(self, dts_content: str) -> Dict[str, DeviceInfo]: diff --git a/src/kerf/init/main.py b/src/kerf/init/main.py index b3135ac..cb819e8 100644 --- a/src/kerf/init/main.py +++ b/src/kerf/init/main.py @@ -49,6 +49,7 @@ GlobalDeviceTree, HardwareInventory, MemoryAllocation, + PoolMemoryRegion, ) @@ -459,8 +460,7 @@ def build_baseline_from_cmdline( memory_allocation = MemoryAllocation( total_bytes=total_bytes, host_reserved_bytes=host_reserved_bytes, - memory_pool_base=memory_pool_base, - memory_pool_bytes=memory_pool_bytes + regions=[PoolMemoryRegion(base=memory_pool_base, size=memory_pool_bytes)] ) device_dict = {} diff --git a/src/kerf/models.py b/src/kerf/models.py index 35667a3..e9a4c21 100644 --- a/src/kerf/models.py +++ b/src/kerf/models.py @@ -16,7 +16,7 @@ Data models for multikernel device tree representation. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import List, Dict, Optional, Set, Tuple from enum import Enum @@ -101,20 +101,45 @@ def get_allocated_cpus(self) -> Set[int]: return set(self.available) - set(self.host_reserved) +@dataclass(frozen=True) +class PoolMemoryRegion: + """One kernel-owned pool chunk as reported by /sys/fs/multikernel/device_tree.""" + + base: int + size: int + node: int = -1 + + @dataclass class MemoryAllocation: - """Memory allocation information.""" + """Pool memory: live chunks from the kernel and per-node sizes requested by the user.""" total_bytes: int host_reserved_bytes: int - memory_pool_base: int - memory_pool_bytes: int + regions: List[PoolMemoryRegion] = field(default_factory=list) + requested: Dict[int, int] = field(default_factory=dict) + + @property + def memory_pool_base(self) -> int: + """Base address of the first live pool chunk, or 0 before any chunks are read back.""" + return self.regions[0].base if self.regions else 0 + + @property + def memory_pool_bytes(self) -> int: + """Total pool size: sum of live chunks, or sum of requested sizes if none yet.""" + if self.regions: + return sum(r.size for r in self.regions) + return sum(self.requested.values()) @property def memory_pool_end(self) -> int: """End address of memory pool.""" return self.memory_pool_base + self.memory_pool_bytes + def bytes_on_node(self, node: int) -> int: + """Total live pool bytes on the given NUMA node.""" + return sum(r.size for r in self.regions if r.node == node) + @dataclass class DeviceInfo: diff --git a/tests/conftest.py b/tests/conftest.py index 087eeaf..12b0918 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,6 +31,7 @@ HardwareInventory, CPUAllocation, MemoryAllocation, + PoolMemoryRegion, DeviceInfo, Instance, InstanceResources, @@ -47,8 +48,7 @@ def sample_hardware(): memory = MemoryAllocation( total_bytes=16 * 1024**3, # 16GB host_reserved_bytes=2 * 1024**3, # 2GB - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, # 14GB + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3, node=0)], # 14GB ) devices = { diff --git a/tests/create_test_dtb.py b/tests/create_test_dtb.py index 60490e8..b63e2d4 100644 --- a/tests/create_test_dtb.py +++ b/tests/create_test_dtb.py @@ -29,6 +29,7 @@ HardwareInventory, CPUAllocation, MemoryAllocation, + PoolMemoryRegion, DeviceInfo, Instance, InstanceResources, @@ -47,8 +48,7 @@ def create_test_tree(): memory = MemoryAllocation( total_bytes=16 * 1024**3, # 16GB host_reserved_bytes=2 * 1024**3, # 2GB - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, # 14GB + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3, node=0)], # 14GB ) devices = { diff --git a/tests/demo.py b/tests/demo.py index e3ee8a6..d236072 100644 --- a/tests/demo.py +++ b/tests/demo.py @@ -29,6 +29,7 @@ HardwareInventory, CPUAllocation, MemoryAllocation, + PoolMemoryRegion, DeviceInfo, Instance, InstanceResources, @@ -50,8 +51,7 @@ def create_demo_system(): memory = MemoryAllocation( total_bytes=16 * 1024**3, # 16GB host_reserved_bytes=2 * 1024**3, # 2GB - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, # 14GB + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3, node=0)], # 14GB ) devices = { diff --git a/tests/test_baseline.py b/tests/test_baseline.py index 1a1a027..7fb924e 100644 --- a/tests/test_baseline.py +++ b/tests/test_baseline.py @@ -60,7 +60,7 @@ def test_validate_baseline_missing_hardware(self): def test_validate_baseline_missing_cpus(self): """Test baseline validation fails without CPU info.""" - from kerf.models import GlobalDeviceTree, HardwareInventory, MemoryAllocation + from kerf.models import GlobalDeviceTree, HardwareInventory, MemoryAllocation, PoolMemoryRegion tree = GlobalDeviceTree( hardware=HardwareInventory( @@ -68,8 +68,7 @@ def test_validate_baseline_missing_cpus(self): memory=MemoryAllocation( total_bytes=16 * 1024**3, host_reserved_bytes=2 * 1024**3, - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3, node=0)], ), devices={}, ), diff --git a/tests/test_kerf.py b/tests/test_kerf.py index e5d5621..a1917e1 100644 --- a/tests/test_kerf.py +++ b/tests/test_kerf.py @@ -29,6 +29,7 @@ HardwareInventory, CPUAllocation, MemoryAllocation, + PoolMemoryRegion, DeviceInfo, Instance, InstanceResources, @@ -49,8 +50,7 @@ def create_test_tree(): memory = MemoryAllocation( total_bytes=16 * 1024**3, # 16GB host_reserved_bytes=2 * 1024**3, # 2GB - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, # 14GB + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3, node=0)], # 14GB ) devices = { diff --git a/tests/test_models.py b/tests/test_models.py index 39631f9..293da50 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -22,6 +22,7 @@ Instance, InstanceResources, MemoryAllocation, + PoolMemoryRegion, ) @@ -53,8 +54,7 @@ def test_memory_allocation_creation(self): memory = MemoryAllocation( total_bytes=16 * 1024**3, host_reserved_bytes=2 * 1024**3, - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3)], ) assert memory.total_bytes == 16 * 1024**3 @@ -67,14 +67,36 @@ def test_memory_pool_end(self): memory = MemoryAllocation( total_bytes=16 * 1024**3, host_reserved_bytes=2 * 1024**3, - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3)], ) expected_end = 0x80000000 + 14 * 1024**3 assert memory.memory_pool_end == expected_end +def test_memory_allocation_derives_pool_from_regions(): + """Pool base/bytes come from the live chunk list when present.""" + mem = MemoryAllocation( + total_bytes=0, + host_reserved_bytes=0, + regions=[ + PoolMemoryRegion(base=0x100000000, size=1 << 30, node=0), + PoolMemoryRegion(base=0x300000000, size=1 << 29, node=1), + ], + ) + assert mem.memory_pool_base == 0x100000000 + assert mem.memory_pool_bytes == (1 << 30) + (1 << 29) + assert mem.bytes_on_node(1) == 1 << 29 + assert mem.bytes_on_node(2) == 0 + + +def test_memory_allocation_derives_pool_from_requested_when_no_regions(): + """Falls back to the requested per-node sizes when no chunks were read back yet.""" + mem = MemoryAllocation(total_bytes=0, host_reserved_bytes=0, requested={-1: 1 << 30}) + assert mem.memory_pool_base == 0 + assert mem.memory_pool_bytes == 1 << 30 + + class TestDeviceInfo: """Test device information model.""" From e2f6b3cc9c3e0d99473a3f262be105f5c0f328c0 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 11:12:14 -0700 Subject: [PATCH 02/23] parser: Read pool chunks and per-node memory requests from the baseline The root device tree now reports pool memory as memory@ nodes with a reg pair and a numa-node-id, and a baseline request names per-node sizes instead of a single base/bytes pair. Parse both shapes, keep cpus- available as the free subset of the pool, reject the legacy memory-base and memory-bytes properties, and emit the request shape from the extractor so the baseline still round-trips through the parser. Signed-off-by: Cong Wang --- src/kerf/dtc/extractor.py | 11 ++- src/kerf/dtc/parser.py | 139 +++++++++++++++++++++++------- src/kerf/dtc/validator.py | 17 ++-- src/kerf/models.py | 3 +- tests/test_baseline.py | 7 +- tests/test_parser.py | 7 +- tests/test_parser_pool.py | 175 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 309 insertions(+), 50 deletions(-) create mode 100644 tests/test_parser_pool.py diff --git a/src/kerf/dtc/extractor.py b/src/kerf/dtc/extractor.py index c7bfb0d..b4d0fdf 100644 --- a/src/kerf/dtc/extractor.py +++ b/src/kerf/dtc/extractor.py @@ -125,9 +125,14 @@ def _add_cpu_properties_sw(self, fdt_sw, cpus): fdt_sw.property("cpus", pack_cpu_ids(cpus.available)) def _add_memory_properties_sw(self, fdt_sw, memory): - """Add memory properties directly to resources node.""" - fdt_sw.property_u64("memory-base", memory.memory_pool_base) - fdt_sw.property_u64("memory-bytes", memory.memory_pool_bytes) + """Add one memory@ request node per requested size (or live chunk).""" + entries = list(memory.requested.items()) or [(r.node, r.size) for r in memory.regions] + for idx, (node, size) in enumerate(entries): + fdt_sw.begin_node(f"memory@{idx}") + fdt_sw.property_u64("size", size) + if node >= 0: + fdt_sw.property_u32("numa-node-id", node) + fdt_sw.end_node() def _add_devices_section_sw(self, fdt_sw, devices): """Add devices section using FdtSw.""" diff --git a/src/kerf/dtc/parser.py b/src/kerf/dtc/parser.py index e484291..842bf3f 100644 --- a/src/kerf/dtc/parser.py +++ b/src/kerf/dtc/parser.py @@ -17,6 +17,7 @@ """ import re +import struct from typing import Dict, List, Optional import libfdt @@ -209,6 +210,11 @@ def _parse_cpu_allocation(self, resources_node: int) -> CPUAllocation: # No cpus property means all CPUs are allocated available = [] + available_free = None + free_prop = self._optional_prop(resources_node, 'cpus-available') + if free_prop is not None: + available_free = unpack_cpu_ids(free_prop) + if available: total = max(available) + 1 else: @@ -218,34 +224,81 @@ def _parse_cpu_allocation(self, resources_node: int) -> CPUAllocation: return CPUAllocation( total=total, host_reserved=host_reserved, - available=available + available=available, + available_free=available_free ) - def _parse_memory_allocation(self, resources_node: int) -> MemoryAllocation: - """Parse memory allocation from resources node.""" + def _subnodes(self, node: int): + """Yield the offsets of a node's immediate children.""" try: - memory_pool_base_prop = self.fdt.getprop(resources_node, 'memory-base') - if len(memory_pool_base_prop) != 8: - raise ParseError(f"Invalid 'memory-base' property size: {len(memory_pool_base_prop)} bytes (expected 8 bytes)") - memory_pool_base = memory_pool_base_prop.as_uint64() - except libfdt.FdtException as e: - raise ParseError(f"Missing 'memory-base' property in /resources: {e}") from e + offset = self.fdt.first_subnode(node) + except libfdt.FdtException: + return + while offset >= 0: + yield offset + try: + offset = self.fdt.next_subnode(offset) + except libfdt.FdtException: + return + def _optional_prop(self, node: int, name: str): + """Return a property, or None when the node does not carry it.""" try: - memory_pool_bytes_prop = self.fdt.getprop(resources_node, 'memory-bytes') - if len(memory_pool_bytes_prop) != 8: - raise ParseError(f"Invalid 'memory-bytes' property size: {len(memory_pool_bytes_prop)} bytes (expected 8 bytes)") - memory_pool_bytes = memory_pool_bytes_prop.as_uint64() - except libfdt.FdtException as e: - raise ParseError(f"Missing 'memory-bytes' property in /resources: {e}") from e + return self.fdt.getprop(node, name) + except libfdt.FdtException: + return None + + def _optional_u32(self, node: int, name: str, default: int) -> int: + """Return a u32 property, or default when the node does not carry it.""" + prop = self._optional_prop(node, name) + if prop is None: + return default + return prop.as_uint32() + + def _optional_u64(self, node: int, name: str) -> Optional[int]: + """Return a u64 property, or None when the node does not carry it.""" + prop = self._optional_prop(node, name) + if prop is None: + return None + return prop.as_uint64() - total_bytes = memory_pool_base + memory_pool_bytes - host_reserved_bytes = 0 + def _parse_memory_allocation(self, resources_node: int) -> MemoryAllocation: + """Parse pool chunks and per-node memory requests from resources node.""" + regions = [] + requested = {} + + for node in self._subnodes(resources_node): + name = self.fdt.get_name(node) + if not name.startswith('memory@'): + continue + node_id = self._optional_u32(node, 'numa-node-id', -1) + reg = self._optional_prop(node, 'reg') + size_prop = self._optional_prop(node, 'size') + if reg is not None: + if len(reg) != 16: + raise ParseError(f"{name}: 'reg' must be two u64 cells") + base, size = struct.unpack('>QQ', bytes(reg)) + regions.append(PoolMemoryRegion(base=base, size=size, node=node_id)) + elif size_prop is not None: + if len(size_prop) != 8: + raise ParseError(f"{name}: 'size' must be a u64") + requested[node_id] = requested.get(node_id, 0) + size_prop.as_uint64() + else: + raise ParseError(f"{name}: expected 'reg' (existing chunk) or 'size' (request)") + + if self._optional_u64(resources_node, 'memory-bytes') is not None: + raise ParseError( + "memory-base/memory-bytes are not supported; " + "use memory@N { size; numa-node-id; }" + ) + + total_bytes = sum(r.size for r in regions) or sum(requested.values()) return MemoryAllocation( total_bytes=total_bytes, - host_reserved_bytes=host_reserved_bytes, - regions=[PoolMemoryRegion(base=memory_pool_base, size=memory_pool_bytes)] + host_reserved_bytes=0, + regions=regions, + requested=requested ) def _parse_devices(self, resources_node: int) -> Dict[str, DeviceInfo]: @@ -773,30 +826,50 @@ def _parse_cpus_from_dts(self, dts_content: str) -> CPUAllocation: ) def _parse_memory_from_dts(self, dts_content: str) -> MemoryAllocation: - """Parse memory allocation from DTS content.""" + """Parse pool chunks and per-node memory requests from DTS content.""" resources_text = self._extract_resources_section(dts_content) if not resources_text: raise ParseError("Missing /resources section in DTS") - memory_base_match = re.search(r'memory-base\s*=\s*<([^>]+)>', resources_text) - if not memory_base_match: - raise ParseError("Missing 'memory-base' property in /resources") - - memory_bytes_match = re.search(r'memory-bytes\s*=\s*<([^>]+)>', resources_text) - if not memory_bytes_match: - raise ParseError("Missing 'memory-bytes' property in /resources") + if re.search(r'memory-(?:base|bytes)\s*=', resources_text): + raise ParseError( + "memory-base/memory-bytes are not supported; " + "use memory@N { size; numa-node-id; }" + ) - memory_pool_base = self._parse_hex_value(memory_base_match.group(1)) - memory_pool_bytes = self._parse_hex_value(memory_bytes_match.group(1)) + regions = [] + requested = {} + + for match in re.finditer(r'(memory@\w+)\s*\{([^}]*)\}', resources_text): + name, body = match.group(1), match.group(2) + node_id = -1 + node_match = re.search(r'numa-node-id\s*=\s*<\s*([^>\s]+)\s*>', body) + if node_match: + node_id = int(node_match.group(1), 0) + + reg_match = re.search(r'reg\s*=\s*<([^>]+)>', body) + size_match = re.search(r'(?]+)>', body) + if reg_match: + cells = reg_match.group(1).split() + if len(cells) != 4: + raise ParseError(f"{name}: 'reg' must be two u64 cells") + base = self._parse_hex_value(' '.join(cells[:2])) + size = self._parse_hex_value(' '.join(cells[2:])) + regions.append(PoolMemoryRegion(base=base, size=size, node=node_id)) + elif size_match: + size = self._parse_hex_value(size_match.group(1)) + requested[node_id] = requested.get(node_id, 0) + size + else: + raise ParseError(f"{name}: expected 'reg' (existing chunk) or 'size' (request)") - total_bytes = memory_pool_base + memory_pool_bytes - host_reserved_bytes = 0 + total_bytes = sum(r.size for r in regions) or sum(requested.values()) return MemoryAllocation( total_bytes=total_bytes, - host_reserved_bytes=host_reserved_bytes, - regions=[PoolMemoryRegion(base=memory_pool_base, size=memory_pool_bytes)] + host_reserved_bytes=0, + regions=regions, + requested=requested ) def _parse_devices_from_dts(self, dts_content: str) -> Dict[str, DeviceInfo]: diff --git a/src/kerf/dtc/validator.py b/src/kerf/dtc/validator.py index 6bcda48..50d8fde 100644 --- a/src/kerf/dtc/validator.py +++ b/src/kerf/dtc/validator.py @@ -254,19 +254,22 @@ def _validate_hardware_inventory(self, tree: GlobalDeviceTree): ) memory = tree.hardware.memory - if memory.total_bytes <= 0: + if memory.total_bytes < 0: self.errors.append("Hardware inventory: Total memory must be positive") if memory.memory_pool_bytes <= 0: self.errors.append("Hardware inventory: Spawn pool size must be positive") + # total_bytes is 0 when the tree only carries pool sizes, not a system total. + if memory.total_bytes and memory.memory_pool_bytes > memory.total_bytes: + self.errors.append("Hardware inventory: Spawn pool extends beyond total memory") + + if not memory.regions: + # A request names sizes only; the kernel picks the chunks, so there + # is nothing to line up with /proc/iomem yet. + return + iomem_pool = get_memory_pool_from_iomem() - if iomem_pool is None or ( - memory.memory_pool_base != iomem_pool[0] or memory.memory_pool_bytes != iomem_pool[1] - ): - # Pool doesn't match kernel-provided pool, validate against total_bytes - if memory.memory_pool_base + memory.memory_pool_bytes > memory.total_bytes: - self.errors.append("Hardware inventory: Spawn pool extends beyond total memory") if iomem_pool is not None: iomem_base, iomem_size = iomem_pool iomem_end = iomem_base + iomem_size diff --git a/src/kerf/models.py b/src/kerf/models.py index e9a4c21..c3fb3b7 100644 --- a/src/kerf/models.py +++ b/src/kerf/models.py @@ -93,8 +93,9 @@ class CPUAllocation: total: int host_reserved: List[int] - available: List[int] + available: List[int] # Pool membership: every CPU in the pool, lent or free topology: Optional[Dict[int, CPUTopology]] = None # CPU ID -> topology info + available_free: Optional[List[int]] = None # Pool members not lent to an instance def get_allocated_cpus(self) -> Set[int]: """Get set of CPUs allocated to instances.""" diff --git a/tests/test_baseline.py b/tests/test_baseline.py index 7fb924e..924b135 100644 --- a/tests/test_baseline.py +++ b/tests/test_baseline.py @@ -101,9 +101,12 @@ def test_write_and_read_baseline(self, sample_hardware): # Verify assert read_tree.hardware.cpus.available == sample_hardware.cpus.available assert ( - read_tree.hardware.memory.memory_pool_base - == sample_hardware.memory.memory_pool_base + read_tree.hardware.memory.memory_pool_bytes + == sample_hardware.memory.memory_pool_bytes ) + assert read_tree.hardware.memory.requested == { + 0: sample_hardware.memory.memory_pool_bytes + } assert len(read_tree.instances) == 0 finally: # Cleanup diff --git a/tests/test_parser.py b/tests/test_parser.py index 3cbdeb3..24ffba4 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -37,10 +37,9 @@ def test_parse_dtb_roundtrip(self, sample_tree): # Verify resources match assert parsed_tree.hardware.cpus.available == sample_tree.hardware.cpus.available - assert ( - parsed_tree.hardware.memory.memory_pool_base - == sample_tree.hardware.memory.memory_pool_base - ) + assert parsed_tree.hardware.memory.requested == { + 0: sample_tree.hardware.memory.memory_pool_bytes + } assert ( parsed_tree.hardware.memory.memory_pool_bytes == sample_tree.hardware.memory.memory_pool_bytes diff --git a/tests/test_parser_pool.py b/tests/test_parser_pool.py new file mode 100644 index 0000000..086db06 --- /dev/null +++ b/tests/test_parser_pool.py @@ -0,0 +1,175 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for parsing pool chunks and per-node memory requests.""" + +import struct + +import libfdt +import pytest + +from kerf.dtc.parser import DeviceTreeParser +from kerf.exceptions import ParseError + + +def _dtb(build): + sw = libfdt.FdtSw() + sw.finish_reservemap() + sw.begin_node("") + sw.property_string("compatible", "linux,multikernel-host") + sw.begin_node("resources") + sw.property("cpus", struct.pack(">QQ", 4, 5)) + build(sw) + sw.end_node() + sw.end_node() + fdt = sw.as_fdt() + fdt.pack() + return bytes(fdt.as_bytearray()) + + +def test_parse_readback_regions(): + def build(sw): + sw.begin_node("memory@100000000") + sw.property_string("device_type", "memory") + sw.property("reg", struct.pack(">QQ", 0x100000000, 1 << 30)) + sw.property_u32("numa-node-id", 1) + sw.end_node() + + tree = DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + mem = tree.hardware.memory + assert [(r.base, r.size, r.node) for r in mem.regions] == [(0x100000000, 1 << 30, 1)] + assert mem.requested == {} + assert mem.total_bytes == 1 << 30 + + +def test_parse_requested_memory_nodes(): + def build(sw): + sw.begin_node("memory@0") + sw.property_u64("size", 1 << 30) + sw.property_u32("numa-node-id", 0) + sw.end_node() + sw.begin_node("memory@1") + sw.property_u64("size", 1 << 29) + sw.end_node() + + tree = DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + assert tree.hardware.memory.regions == [] + assert tree.hardware.memory.requested == {0: 1 << 30, -1: 1 << 29} + assert tree.hardware.memory.total_bytes == (1 << 30) + (1 << 29) + + +def test_parse_memory_node_without_reg_or_size_rejected(): + def build(sw): + sw.begin_node("memory@0") + sw.property_u32("numa-node-id", 0) + sw.end_node() + + with pytest.raises(ParseError): + DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + + +def test_parse_legacy_memory_base_bytes_rejected(): + def build(sw): + sw.property_u64("memory-base", 0x80000000) + sw.property_u64("memory-bytes", 1 << 30) + + with pytest.raises(ParseError): + DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + + +def test_parse_cpus_available(): + def build(sw): + sw.property("cpus-available", struct.pack(">Q", 5)) + sw.begin_node("memory@0") + sw.property_u64("size", 1 << 30) + sw.end_node() + + tree = DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + assert tree.hardware.cpus.available == [4, 5] + assert tree.hardware.cpus.available_free == [5] + + +def test_parse_cpus_available_absent(): + def build(sw): + sw.begin_node("memory@0") + sw.property_u64("size", 1 << 30) + sw.end_node() + + tree = DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + assert tree.hardware.cpus.available_free is None + + +_DTS_REQUEST = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + + memory@0 { + size = <0x0 0x40000000>; + numa-node-id = <1>; + }; + + memory@1 { + size = <0x0 0x20000000>; + }; + }; +}; +""" + +_DTS_READBACK = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + + memory@100000000 { + device_type = "memory"; + reg = <0x1 0x0 0x0 0x40000000>; + numa-node-id = <1>; + }; + }; +}; +""" + +_DTS_LEGACY = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + memory-base = <0x0 0x80000000>; + memory-bytes = <0x0 0x40000000>; + }; +}; +""" + + +def test_parse_dts_requested_memory_nodes(): + tree = DeviceTreeParser().parse_dts(_DTS_REQUEST) + assert tree.hardware.memory.regions == [] + assert tree.hardware.memory.requested == {1: 1 << 30, -1: 1 << 29} + + +def test_parse_dts_readback_regions(): + tree = DeviceTreeParser().parse_dts(_DTS_READBACK) + regions = tree.hardware.memory.regions + assert [(r.base, r.size, r.node) for r in regions] == [(0x100000000, 1 << 30, 1)] + + +def test_parse_dts_legacy_memory_base_bytes_rejected(): + with pytest.raises(ParseError): + DeviceTreeParser().parse_dts(_DTS_LEGACY) From b64e634a182d5965729da98983cadbb2cb6ca8c0 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 11:24:11 -0700 Subject: [PATCH 03/23] parser: Scope legacy memory checks to /resources and migrate the examples A NUMA topology node describes itself with memory-base and memory-size, so scanning the whole /resources body for the legacy pool properties rejected trees that were already migrated. Split the body into its own properties and its direct children, check only the former, read pool chunks only from depth-0 memory@ nodes, migrate the example baselines to memory@N requests, reject a /resources that describes no memory at all, and catch a lone memory-base on the DTB path. Signed-off-by: Cong Wang --- examples/bad_system.dts | 5 ++- examples/baseline.dts | 5 ++- examples/conflict_example.dts | 5 ++- examples/edge_computing.dts | 5 ++- examples/high_performance.dts | 5 ++- examples/minimal.dts | 5 ++- examples/numa_topology.dts | 21 ++++++++- examples/simple_numa.dts | 11 ++++- examples/system.dts | 5 ++- src/kerf/dtc/parser.py | 72 ++++++++++++++++++++++++------ src/kerf/dtc/validator.py | 13 +++--- tests/test_parser_pool.py | 83 +++++++++++++++++++++++++++++++++++ 12 files changed, 199 insertions(+), 36 deletions(-) diff --git a/examples/bad_system.dts b/examples/bad_system.dts index 478ca89..7f113e5 100644 --- a/examples/bad_system.dts +++ b/examples/bad_system.dts @@ -14,8 +14,9 @@ resources { cpus = <4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31>; - memory-base = <0x80000000>; - memory-bytes = <0x0 0x380000000>; // 14GB + memory@0 { + size = <0x0 0x380000000>; // 14GB + }; devices { eth0: ethernet@0 { diff --git a/examples/baseline.dts b/examples/baseline.dts index 88c753e..5cce70d 100644 --- a/examples/baseline.dts +++ b/examples/baseline.dts @@ -20,8 +20,9 @@ resources { cpus = <4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31>; - memory-base = <0x80000000>; - memory-bytes = <0x0 0x380000000>; // 14GB + memory@0 { + size = <0x0 0x380000000>; // 14GB + }; devices { eth0: ethernet@0 { diff --git a/examples/conflict_example.dts b/examples/conflict_example.dts index a9cd88b..a56e87d 100644 --- a/examples/conflict_example.dts +++ b/examples/conflict_example.dts @@ -13,7 +13,8 @@ resources { cpus = <2 3 4 5 6 7 8 9 10 11 12 13 14 15>; - memory-base = <0x100000000>; - memory-bytes = <0x0 0x300000000>; // 12GB + memory@0 { + size = <0x0 0x300000000>; // 12GB + }; }; }; diff --git a/examples/edge_computing.dts b/examples/edge_computing.dts index 5e2a73f..f859d3b 100644 --- a/examples/edge_computing.dts +++ b/examples/edge_computing.dts @@ -12,8 +12,9 @@ resources { cpus = <2 3 4 5 6 7 8 9 10 11 12 13 14 15>; - memory-base = <0x200000000>; - memory-bytes = <0x0 0x600000000>; // 24GB + memory@0 { + size = <0x0 0x600000000>; // 24GB + }; devices { eth0: ethernet@0 { diff --git a/examples/high_performance.dts b/examples/high_performance.dts index 4001623..46b4783 100644 --- a/examples/high_performance.dts +++ b/examples/high_performance.dts @@ -18,8 +18,9 @@ 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119>; - memory-base = <0x400000000>; - memory-bytes = <0x0 0xC00000000>; // 48GB + memory@0 { + size = <0x0 0xC00000000>; // 48GB + }; devices { eth0: ethernet@0 { diff --git a/examples/minimal.dts b/examples/minimal.dts index ab6b2b5..10ca926 100644 --- a/examples/minimal.dts +++ b/examples/minimal.dts @@ -12,7 +12,8 @@ resources { cpus = <2 3 4 5 6 7>; - memory-base = <0x40000000>; - memory-bytes = <0x0 0x1C0000000>; // 7GB + memory@0 { + size = <0x0 0x1C0000000>; // 7GB + }; }; }; diff --git a/examples/numa_topology.dts b/examples/numa_topology.dts index 1424653..a396824 100644 --- a/examples/numa_topology.dts +++ b/examples/numa_topology.dts @@ -58,8 +58,25 @@ }; }; - memory-base = <0x0 0x800000000>; - memory-bytes = <0x0 0x1800000000>; // 96GB + memory@0 { + size = <0x0 0x600000000>; // 24GB + numa-node-id = <0>; + }; + + memory@1 { + size = <0x0 0x600000000>; // 24GB + numa-node-id = <1>; + }; + + memory@2 { + size = <0x0 0x600000000>; // 24GB + numa-node-id = <2>; + }; + + memory@3 { + size = <0x0 0x600000000>; // 24GB + numa-node-id = <3>; + }; devices { eth0: ethernet@0 { diff --git a/examples/simple_numa.dts b/examples/simple_numa.dts index 59e08fa..fcf1c3b 100644 --- a/examples/simple_numa.dts +++ b/examples/simple_numa.dts @@ -32,8 +32,15 @@ }; }; - memory-base = <0x0 0x400000000>; - memory-bytes = <0x0 0xC00000000>; // 48GB + memory@0 { + size = <0x0 0x600000000>; // 24GB + numa-node-id = <0>; + }; + + memory@1 { + size = <0x0 0x600000000>; // 24GB + numa-node-id = <1>; + }; devices { eth0: ethernet@0 { diff --git a/examples/system.dts b/examples/system.dts index 96b89e7..c540734 100644 --- a/examples/system.dts +++ b/examples/system.dts @@ -13,8 +13,9 @@ resources { cpus = <4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31>; - memory-base = <0x80000000>; - memory-bytes = <0x0 0x380000000>; // 14GB + memory@0 { + size = <0x0 0x380000000>; // 14GB + }; devices { eth0: ethernet@0 { diff --git a/src/kerf/dtc/parser.py b/src/kerf/dtc/parser.py index 842bf3f..0eac0ab 100644 --- a/src/kerf/dtc/parser.py +++ b/src/kerf/dtc/parser.py @@ -18,7 +18,7 @@ import re import struct -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple import libfdt @@ -39,6 +39,17 @@ ) +_LEGACY_MEMORY_ERROR = ( + "memory-base/memory-bytes are not supported; " + "use memory@N { size; numa-node-id; }" +) + +_NO_MEMORY_ERROR = ( + "No memory description in /resources; " + "expected memory@N { size; numa-node-id; }" +) + + class DeviceTreeParser: """Parser for multikernel device trees.""" @@ -286,11 +297,12 @@ def _parse_memory_allocation(self, resources_node: int) -> MemoryAllocation: else: raise ParseError(f"{name}: expected 'reg' (existing chunk) or 'size' (request)") - if self._optional_u64(resources_node, 'memory-bytes') is not None: - raise ParseError( - "memory-base/memory-bytes are not supported; " - "use memory@N { size; numa-node-id; }" - ) + for legacy in ('memory-base', 'memory-bytes'): + if self._optional_prop(resources_node, legacy) is not None: + raise ParseError(_LEGACY_MEMORY_ERROR) + + if not regions and not requested: + raise ParseError(_NO_MEMORY_ERROR) total_bytes = sum(r.size for r in regions) or sum(requested.values()) @@ -825,6 +837,38 @@ def _parse_cpus_from_dts(self, dts_content: str) -> CPUAllocation: topology=topology ) + def _split_node_body(self, body: str) -> Tuple[str, List[Tuple[str, str]]]: + """Split a DTS node body into its own property text and its direct children. + + Nested nodes carry properties of their own (a NUMA node@N describes itself + with memory-base/memory-size), so callers must not confuse them with the + properties of the node they are inspecting. + """ + properties = [] + children = [] + depth = 0 + chunk_start = 0 + body_start = 0 + name = "" + + for i, char in enumerate(body): + if char == '{': + depth += 1 + if depth == 1: + head = body[chunk_start:i] + tokens = head.replace(';', ' ').split() + name = tokens[-1] if tokens else "" + properties.append(head[:head.rfind(name)] if name else head) + body_start = i + 1 + elif char == '}': + depth -= 1 + if depth == 0: + children.append((name, body[body_start:i])) + chunk_start = i + 1 + + properties.append(body[chunk_start:]) + return '\n'.join(properties), children + def _parse_memory_from_dts(self, dts_content: str) -> MemoryAllocation: """Parse pool chunks and per-node memory requests from DTS content.""" @@ -832,17 +876,16 @@ def _parse_memory_from_dts(self, dts_content: str) -> MemoryAllocation: if not resources_text: raise ParseError("Missing /resources section in DTS") - if re.search(r'memory-(?:base|bytes)\s*=', resources_text): - raise ParseError( - "memory-base/memory-bytes are not supported; " - "use memory@N { size; numa-node-id; }" - ) + own_properties, children = self._split_node_body(resources_text) + if re.search(r'memory-(?:base|bytes)\s*=', own_properties): + raise ParseError(_LEGACY_MEMORY_ERROR) regions = [] requested = {} - for match in re.finditer(r'(memory@\w+)\s*\{([^}]*)\}', resources_text): - name, body = match.group(1), match.group(2) + for name, body in children: + if not name.startswith('memory@'): + continue node_id = -1 node_match = re.search(r'numa-node-id\s*=\s*<\s*([^>\s]+)\s*>', body) if node_match: @@ -863,6 +906,9 @@ def _parse_memory_from_dts(self, dts_content: str) -> MemoryAllocation: else: raise ParseError(f"{name}: expected 'reg' (existing chunk) or 'size' (request)") + if not regions and not requested: + raise ParseError(_NO_MEMORY_ERROR) + total_bytes = sum(r.size for r in regions) or sum(requested.values()) return MemoryAllocation( diff --git a/src/kerf/dtc/validator.py b/src/kerf/dtc/validator.py index 50d8fde..edca1d7 100644 --- a/src/kerf/dtc/validator.py +++ b/src/kerf/dtc/validator.py @@ -260,15 +260,18 @@ def _validate_hardware_inventory(self, tree: GlobalDeviceTree): if memory.memory_pool_bytes <= 0: self.errors.append("Hardware inventory: Spawn pool size must be positive") - # total_bytes is 0 when the tree only carries pool sizes, not a system total. + # Only bites a caller-built tree: a parsed one derives total_bytes from + # the pool itself, and 0 means the caller knows no system total. if memory.total_bytes and memory.memory_pool_bytes > memory.total_bytes: self.errors.append("Hardware inventory: Spawn pool extends beyond total memory") - if not memory.regions: - # A request names sizes only; the kernel picks the chunks, so there - # is nothing to line up with /proc/iomem yet. - return + # A request names sizes only; the kernel picks the chunks, so there is + # nothing to line up with /proc/iomem until they are read back. + if memory.regions: + self._validate_pool_against_iomem(memory) + def _validate_pool_against_iomem(self, memory): + """Compare the pool chunks read back from the kernel with /proc/iomem.""" iomem_pool = get_memory_pool_from_iomem() if iomem_pool is not None: iomem_base, iomem_size = iomem_pool diff --git a/tests/test_parser_pool.py b/tests/test_parser_pool.py index 086db06..d4a6f91 100644 --- a/tests/test_parser_pool.py +++ b/tests/test_parser_pool.py @@ -15,6 +15,7 @@ """Tests for parsing pool chunks and per-node memory requests.""" import struct +from pathlib import Path import libfdt import pytest @@ -173,3 +174,85 @@ def test_parse_dts_readback_regions(): def test_parse_dts_legacy_memory_base_bytes_rejected(): with pytest.raises(ParseError): DeviceTreeParser().parse_dts(_DTS_LEGACY) + + +_DTS_TOPOLOGY = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + + topology { + numa-nodes { + node@0 { + node-id = <0>; + memory-base = <0x0 0x0>; + memory-size = <0x0 0x800000000>; + cpus = <4>; + }; + + node@1 { + node-id = <1>; + memory-base = <0x0 0x800000000>; + memory-size = <0x0 0x800000000>; + cpus = <5>; + }; + }; + }; + + memory@0 { + size = <0x0 0x40000000>; + numa-node-id = <1>; + }; + }; +}; +""" + +_DTS_NO_MEMORY = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + }; +}; +""" + + +def test_parse_dts_topology_memory_base_is_not_the_pool(): + tree = DeviceTreeParser().parse_dts(_DTS_TOPOLOGY) + assert tree.hardware.memory.requested == {1: 1 << 30} + assert tree.hardware.memory.regions == [] + + +def test_parse_dts_without_memory_rejected(): + with pytest.raises(ParseError, match="No memory description"): + DeviceTreeParser().parse_dts(_DTS_NO_MEMORY) + + +def test_parse_dtb_without_memory_rejected(): + def build(sw): + sw.property_u32("placeholder", 0) + + with pytest.raises(ParseError, match="No memory description"): + DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + + +def test_parse_dtb_legacy_memory_base_alone_rejected(): + def build(sw): + sw.property_u64("memory-base", 0x80000000) + sw.begin_node("memory@0") + sw.property_u64("size", 1 << 30) + sw.end_node() + + with pytest.raises(ParseError, match="not supported"): + DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + + +@pytest.mark.parametrize( + "path", sorted(str(p) for p in Path(__file__).parent.parent.glob("examples/*.dts")) +) +def test_parse_example_dts(path): + tree = DeviceTreeParser().parse_dts(Path(path).read_text(encoding="utf-8")) + assert tree.hardware.memory.memory_pool_bytes > 0 From 61144b8ece530d0875d9026744dfb3da0777d2b1 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 11:30:59 -0700 Subject: [PATCH 04/23] extractor: Describe baseline memory as per-node allocation requests The baseline written to the kernel is a request, not a description of memory the host already set aside, so the extractor emits one memory@N node per requested node carrying a size and, when the node is explicit, a numa-node-id, exactly the shape baseline.c parses. This commit adds the test that pins that format; the emitter itself landed with the parser change so the suite never saw a broken round-trip. Signed-off-by: Cong Wang --- tests/test_baseline.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_baseline.py b/tests/test_baseline.py index 924b135..c730eb8 100644 --- a/tests/test_baseline.py +++ b/tests/test_baseline.py @@ -134,3 +134,26 @@ def test_write_baseline_invalid_tree(self, sample_tree): finally: if os.path.exists(baseline_path): os.unlink(baseline_path) + + def test_baseline_emits_memory_requests(self): + import libfdt + from kerf.models import GlobalDeviceTree, HardwareInventory, CPUAllocation, MemoryAllocation + from kerf.dtc.extractor import InstanceExtractor + + hw = HardwareInventory( + cpus=CPUAllocation(total=8, host_reserved=[0, 1], available=[4, 5]), + memory=MemoryAllocation(total_bytes=0, host_reserved_bytes=0, requested={0: 1 << 30, -1: 1 << 29}), + devices={}, + ) + dtb = InstanceExtractor().generate_global_dtb(GlobalDeviceTree(hardware=hw, instances={}, device_references={})) + fdt = libfdt.Fdt(dtb) + res = fdt.path_offset("/resources") + m0 = fdt.subnode_offset(res, "memory@0") + assert fdt.getprop(m0, "size").as_uint64() == 1 << 30 + assert fdt.getprop(m0, "numa-node-id").as_uint32() == 0 + m1 = fdt.subnode_offset(res, "memory@1") + assert fdt.getprop(m1, "size").as_uint64() == 1 << 29 + with pytest.raises(libfdt.FdtException): + fdt.getprop(m1, "numa-node-id") + with pytest.raises(libfdt.FdtException): + fdt.getprop(res, "memory-base") From d2c9874ba974ff93250300c28b8df80c65df4e70 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 11:32:31 -0700 Subject: [PATCH 05/23] pool: Compute the difference between the live pool and a requested baseline Re-initializing the pool means moving resources between the host and the pool rather than writing a new baseline, so kerf needs to know what differs. Add compute_pool_diff(): set differences for CPUs and PCI devices, and a per-node memory comparison that grows a node by the missing amount or releases whole chunks when it holds too much, preferring chunks with no instance allocation and never splitting one. The request is the desired state, so memory on nodes it does not mention counts as surplus. Signed-off-by: Cong Wang --- src/kerf/pool_diff.py | 95 +++++++++++++++++++++++++++++++++++++++++ tests/test_pool_diff.py | 73 +++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 src/kerf/pool_diff.py create mode 100644 tests/test_pool_diff.py diff --git a/src/kerf/pool_diff.py b/src/kerf/pool_diff.py new file mode 100644 index 0000000..96c891e --- /dev/null +++ b/src/kerf/pool_diff.py @@ -0,0 +1,95 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Difference between the live resource pool and a requested baseline.""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Set, Tuple + +from .models import GlobalDeviceTree, PoolMemoryRegion + +ANY_NODE = -1 + + +@dataclass +class PoolDiff: + cpus_to_pool: List[int] = field(default_factory=list) + cpus_to_host: List[int] = field(default_factory=list) + devices_to_pool: List[str] = field(default_factory=list) + devices_to_host: List[str] = field(default_factory=list) + memory_to_pool: List[Tuple[int, int]] = field(default_factory=list) + memory_to_host: List[PoolMemoryRegion] = field(default_factory=list) + + def is_empty(self) -> bool: + return not any((self.cpus_to_pool, self.cpus_to_host, self.devices_to_pool, + self.devices_to_host, self.memory_to_pool, self.memory_to_host)) + + +def _pci_ids(tree: GlobalDeviceTree) -> Set[str]: + return {d.pci_id for d in tree.hardware.devices.values() if d.pci_id} + + +def _memory_diff(regions: List[PoolMemoryRegion], requested: Dict[int, int], + busy: Set[int], diff: PoolDiff) -> None: + remaining = list(regions) + + for node, want in requested.items(): + if node == ANY_NODE: + continue + have = sum(r.size for r in remaining if r.node == node) + if want > have: + diff.memory_to_pool.append((node, want - have)) + elif want < have: + _release(remaining, lambda r, node=node: r.node == node, have - want, busy, diff) + + # Chunks on nodes nobody asked for are surplus, unless an any-node request absorbs them. + explicit = {n for n in requested if n != ANY_NODE} + any_want = requested.get(ANY_NODE, 0) + unclaimed = [r for r in remaining if r.node not in explicit] + have = sum(r.size for r in unclaimed) + if any_want > have: + diff.memory_to_pool.append((ANY_NODE, any_want - have)) + elif any_want < have: + _release(remaining, lambda r: r.node not in explicit, have - any_want, busy, diff) + + +def _release(remaining: List[PoolMemoryRegion], pred, surplus: int, + busy: Set[int], diff: PoolDiff) -> None: + candidates = sorted((r for r in remaining if pred(r)), + key=lambda r: (r.base in busy, -r.size)) + for r in candidates: + if surplus <= 0: + break + if r.size > surplus: + continue + diff.memory_to_host.append(r) + remaining.remove(r) + surplus -= r.size + + +def compute_pool_diff(current: GlobalDeviceTree, requested: GlobalDeviceTree, + busy_chunks: Optional[Set[int]] = None) -> PoolDiff: + diff = PoolDiff() + cur_cpus = set(current.hardware.cpus.available) + req_cpus = set(requested.hardware.cpus.available) + diff.cpus_to_pool = sorted(req_cpus - cur_cpus) + diff.cpus_to_host = sorted(cur_cpus - req_cpus) + + cur_dev, req_dev = _pci_ids(current), _pci_ids(requested) + diff.devices_to_pool = sorted(req_dev - cur_dev) + diff.devices_to_host = sorted(cur_dev - req_dev) + + _memory_diff(current.hardware.memory.regions, requested.hardware.memory.requested, + busy_chunks or set(), diff) + return diff diff --git a/tests/test_pool_diff.py b/tests/test_pool_diff.py new file mode 100644 index 0000000..4b44807 --- /dev/null +++ b/tests/test_pool_diff.py @@ -0,0 +1,73 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Tests for computing the difference between the live pool and a requested baseline. +""" + +from kerf.models import (CPUAllocation, GlobalDeviceTree, HardwareInventory, + MemoryAllocation, PoolMemoryRegion, DeviceInfo) +from kerf.pool_diff import compute_pool_diff + +GB = 1 << 30 + + +def _tree(cpus, regions=None, requested=None, devices=()): + hw = HardwareInventory( + cpus=CPUAllocation(total=16, host_reserved=[0], available=list(cpus)), + memory=MemoryAllocation(total_bytes=0, host_reserved_bytes=0, + regions=list(regions or []), requested=dict(requested or {})), + devices={d: DeviceInfo(name=d, compatible="pci", pci_id=d) for d in devices}, + ) + return GlobalDeviceTree(hardware=hw, instances={}, device_references={}) + + +def test_cpu_and_device_set_differences(): + cur = _tree([4, 5, 6], devices=["0000:03:00.0"]) + req = _tree([5, 6, 7], devices=["0000:04:00.0"]) + d = compute_pool_diff(cur, req) + assert d.cpus_to_pool == [7] + assert d.cpus_to_host == [4] + assert d.devices_to_pool == ["0000:04:00.0"] + assert d.devices_to_host == ["0000:03:00.0"] + + +def test_memory_grow_per_node(): + cur = _tree([4], regions=[PoolMemoryRegion(0x1_0000_0000, GB, 0)]) + req = _tree([4], requested={0: 2 * GB, 1: GB}) + d = compute_pool_diff(cur, req) + assert sorted(d.memory_to_pool) == [(0, GB), (1, GB)] + assert d.memory_to_host == [] + + +def test_memory_shrink_prefers_idle_chunks(): + a = PoolMemoryRegion(0x1_0000_0000, GB, 0) + b = PoolMemoryRegion(0x2_0000_0000, GB, 0) + cur = _tree([4], regions=[a, b]) + req = _tree([4], requested={0: GB}) + d = compute_pool_diff(cur, req, busy_chunks={a.base}) + assert d.memory_to_host == [b] + assert d.memory_to_pool == [] + + +def test_any_node_request_matches_total(): + cur = _tree([4], regions=[PoolMemoryRegion(0x1_0000_0000, GB, 0), PoolMemoryRegion(0x2_0000_0000, GB, 1)]) + req = _tree([4], requested={-1: 2 * GB}) + assert compute_pool_diff(cur, req).is_empty() + + +def test_identical_state_is_empty(): + cur = _tree([4, 5], regions=[PoolMemoryRegion(0x1_0000_0000, GB, 0)]) + req = _tree([4, 5], requested={0: GB}) + assert compute_pool_diff(cur, req).is_empty() From 9da9520fa7689c94560020ac94a8f522c6d19ae0 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 11:37:35 -0700 Subject: [PATCH 06/23] pool: Cover surplus-node release and size ordering in the diff The most consequential behaviour of the pool diff, that a request naming only some nodes releases the memory on every other node, had no test, nor did the larger-first order in which idle chunks are returned. Add both cases and state the desired-state rule in the docstring so a future change cannot silently turn it into a merge. Signed-off-by: Cong Wang --- src/kerf/pool_diff.py | 1 + tests/test_pool_diff.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/kerf/pool_diff.py b/src/kerf/pool_diff.py index 96c891e..ce4d814 100644 --- a/src/kerf/pool_diff.py +++ b/src/kerf/pool_diff.py @@ -80,6 +80,7 @@ def _release(remaining: List[PoolMemoryRegion], pred, surplus: int, def compute_pool_diff(current: GlobalDeviceTree, requested: GlobalDeviceTree, busy_chunks: Optional[Set[int]] = None) -> PoolDiff: + """The request is the desired state; memory on nodes it omits counts as surplus.""" diff = PoolDiff() cur_cpus = set(current.hardware.cpus.available) req_cpus = set(requested.hardware.cpus.available) diff --git a/tests/test_pool_diff.py b/tests/test_pool_diff.py index 4b44807..243d7c9 100644 --- a/tests/test_pool_diff.py +++ b/tests/test_pool_diff.py @@ -71,3 +71,22 @@ def test_identical_state_is_empty(): cur = _tree([4, 5], regions=[PoolMemoryRegion(0x1_0000_0000, GB, 0)]) req = _tree([4, 5], requested={0: GB}) assert compute_pool_diff(cur, req).is_empty() + + +def test_unrequested_node_memory_is_surplus(): + a = PoolMemoryRegion(0x1_0000_0000, GB, 0) + cur = _tree([4], regions=[a]) + req = _tree([4], requested={1: GB}) + d = compute_pool_diff(cur, req) + assert d.memory_to_pool == [(1, GB)] + assert d.memory_to_host == [a] + + +def test_shrink_prefers_larger_idle_chunks(): + small = PoolMemoryRegion(0x1_0000_0000, GB // 2, 0) + big = PoolMemoryRegion(0x2_0000_0000, GB, 0) + cur = _tree([4], regions=[small, big]) + req = _tree([4], requested={0: GB // 2}) + d = compute_pool_diff(cur, req) + assert d.memory_to_host == [big] + assert d.memory_to_pool == [] From 203bee03f1bff08fab019342ea152e598536490e Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 11:43:24 -0700 Subject: [PATCH 07/23] overlay: Address fragments by target path and generate pool transactions The kernel dropped the mk,instance property and now addresses every overlay fragment by the standard target-path: /resources for the pool, /instances/ for an instance and /instances for create and remove. Generate pool transactions from a PoolDiff as a /resources fragment whose operations read from the pool's point of view, convert the instance update and create overlays to target-path, rename the memory items to memory@N and the NUMA property to numa-node-id, and carry numa- node-id on an instance's memory-add so a grow of a node-pinned instance stays on its node. Signed-off-by: Cong Wang --- src/kerf/dtc/overlay.py | 234 +++++++++++++++++-------------------- tests/test_pool_overlay.py | 151 ++++++++++++++++++++++++ 2 files changed, 259 insertions(+), 126 deletions(-) create mode 100644 tests/test_pool_overlay.py diff --git a/src/kerf/dtc/overlay.py b/src/kerf/dtc/overlay.py index a0e6829..44d22e8 100644 --- a/src/kerf/dtc/overlay.py +++ b/src/kerf/dtc/overlay.py @@ -19,17 +19,35 @@ overlays (DTBO) that represent incremental changes to the device tree state. """ -from typing import Set +import struct +from typing import Optional, Set, Tuple import libfdt from ..models import GlobalDeviceTree +from ..pool_diff import PoolDiff from .cells import pack_cpu_id, pack_cpu_ids +Range = Optional[Tuple[int, int]] + + +def _memory_ranges(old_base: int, old_size: int, new_base: int, new_size: int) -> Tuple[Range, Range]: + """Ranges to take back from and to hand to an instance whose memory changed.""" + if old_base != new_base: + return (old_base, old_size), (new_base, new_size) + if new_size > old_size: + return None, (old_base + old_size, new_size - old_size) + if new_size < old_size: + return (old_base + new_size, old_size - new_size), None + return None, None + class OverlayGenerator: """Generates device tree overlay blobs (DTBO) from device tree model deltas.""" + POOL_PATH = "/resources" + INSTANCES_PATH = "/instances" + def generate_overlay(self, current: GlobalDeviceTree, modified: GlobalDeviceTree) -> bytes: """ Generate overlay DTBO representing the difference between current and modified states. @@ -92,8 +110,6 @@ def generate_update_overlay(self, instance_name: str, old_instance, new_instance Returns: DTBO blob as bytes containing resource update operations """ - import struct - fdt_sw = libfdt.FdtSw() fdt_sw.finish_reservemap() @@ -105,11 +121,16 @@ def generate_update_overlay(self, instance_name: str, old_instance, new_instance cpus_to_remove = sorted(old_cpus - new_cpus) cpus_to_add = sorted(new_cpus - old_cpus) - old_mem_base = old_instance.resources.memory_base - old_mem_size = old_instance.resources.memory_bytes - new_mem_base = new_instance.resources.memory_base - new_mem_size = new_instance.resources.memory_bytes - memory_changed = (old_mem_base != new_mem_base) or (old_mem_size != new_mem_size) + memory_to_remove, memory_to_add = _memory_ranges( + old_instance.resources.memory_base, + old_instance.resources.memory_bytes, + new_instance.resources.memory_base, + new_instance.resources.memory_bytes, + ) + + numa_node = None + if new_instance.resources.numa_nodes: + numa_node = new_instance.resources.numa_nodes[0] old_devices = set(old_instance.resources.devices) new_devices = set(new_instance.resources.devices) @@ -118,160 +139,121 @@ def generate_update_overlay(self, instance_name: str, old_instance, new_instance # Single fragment with all operations fdt_sw.begin_node("fragment@0") + fdt_sw.property_string("target-path", f"{self.INSTANCES_PATH}/{instance_name}") fdt_sw.begin_node("__overlay__") - # 1. memory-remove (if memory shrunk or base changed) - if memory_changed: - if old_mem_base == new_mem_base: - # Same base: only remove the excess if shrinking - if old_mem_size > new_mem_size: - remove_base = old_mem_base + new_mem_size - remove_size = old_mem_size - new_mem_size - fdt_sw.begin_node("memory-remove") - fdt_sw.property_string("mk,instance", instance_name) - fdt_sw.begin_node("region@0") - reg_data = struct.pack(">QQ", remove_base, remove_size) - fdt_sw.property("reg", reg_data) - fdt_sw.end_node() - fdt_sw.end_node() - else: - # Different base: remove entire old region - fdt_sw.begin_node("memory-remove") - fdt_sw.property_string("mk,instance", instance_name) - fdt_sw.begin_node("region@0") - reg_data = struct.pack(">QQ", old_mem_base, old_mem_size) - fdt_sw.property("reg", reg_data) - fdt_sw.end_node() - fdt_sw.end_node() + if memory_to_remove: + fdt_sw.begin_node("memory-remove") + self._memory_item(fdt_sw, memory_to_remove) + fdt_sw.end_node() - # 2. memory-add (if memory grew or base changed) - if memory_changed: - if old_mem_base == new_mem_base: - # Same base: only add the extension if growing - if new_mem_size > old_mem_size: - add_base = old_mem_base + old_mem_size - add_size = new_mem_size - old_mem_size - fdt_sw.begin_node("memory-add") - fdt_sw.property_string("mk,instance", instance_name) - fdt_sw.begin_node("region@0") - reg_data = struct.pack(">QQ", add_base, add_size) - fdt_sw.property("reg", reg_data) - fdt_sw.end_node() - fdt_sw.end_node() - else: - # Different base: add entire new region - fdt_sw.begin_node("memory-add") - fdt_sw.property_string("mk,instance", instance_name) - fdt_sw.begin_node("region@0") - reg_data = struct.pack(">QQ", new_mem_base, new_mem_size) - fdt_sw.property("reg", reg_data) - fdt_sw.end_node() - fdt_sw.end_node() + if memory_to_add: + fdt_sw.begin_node("memory-add") + self._memory_item(fdt_sw, memory_to_add, numa_node) + fdt_sw.end_node() - # 3. cpu-remove (if CPUs removed) - if cpus_to_remove: - fdt_sw.begin_node("cpu-remove") - fdt_sw.property_string("mk,instance", instance_name) + self._cpu_op(fdt_sw, "cpu-remove", cpus_to_remove) + self._cpu_op(fdt_sw, "cpu-add", cpus_to_add, numa_node) + self._device_op(fdt_sw, "device-remove", devices_to_remove) + self._device_op(fdt_sw, "device-add", devices_to_add) - for cpu_id in cpus_to_remove: - fdt_sw.begin_node(f"cpu@{cpu_id}") - fdt_sw.property("reg", pack_cpu_id(cpu_id)) - fdt_sw.end_node() + fdt_sw.end_node() # End __overlay__ + fdt_sw.end_node() # End fragment@0 - fdt_sw.end_node() + fdt_sw.end_node() # End root - # 4. cpu-add (if CPUs added) - if cpus_to_add: - fdt_sw.begin_node("cpu-add") - fdt_sw.property_string("mk,instance", instance_name) + dtb = fdt_sw.as_fdt() + dtb.pack() + return dtb.as_bytearray() - for cpu_id in cpus_to_add: - fdt_sw.begin_node(f"cpu@{cpu_id}") - fdt_sw.property("reg", pack_cpu_id(cpu_id)) + def generate_pool_overlay(self, diff: PoolDiff) -> bytes: + """ + Generate an overlay moving resources between the host and the pool. - if new_instance.resources.numa_nodes: - fdt_sw.property_u32("numa-node", new_instance.resources.numa_nodes[0]) + Operation names read from the pool's point of view: memory-add grows the + pool, cpu-remove returns a pool CPU to the host, and so on. - fdt_sw.end_node() + Args: + diff: Resources to move, as computed against the requested baseline - fdt_sw.end_node() + Returns: + DTBO blob as bytes containing a single fragment targeting /resources + """ + fdt_sw = libfdt.FdtSw() + fdt_sw.finish_reservemap() - # 5. device-remove (if devices removed) - if devices_to_remove: - fdt_sw.begin_node("device-remove") - fdt_sw.property_string("mk,instance", instance_name) + fdt_sw.begin_node("") + fdt_sw.property_string("compatible", "linux,multikernel-overlay") - for idx, pci_id in enumerate(devices_to_remove): - fdt_sw.begin_node(f"pci@{idx}") - fdt_sw.property_string("pci-id", pci_id) - fdt_sw.end_node() + fdt_sw.begin_node("fragment@0") + fdt_sw.property_string("target-path", self.POOL_PATH) + fdt_sw.begin_node("__overlay__") + if diff.memory_to_host: + fdt_sw.begin_node("memory-remove") + for idx, region in enumerate(diff.memory_to_host): + fdt_sw.begin_node(f"memory@{idx}") + fdt_sw.property("reg", struct.pack(">QQ", region.base, region.size)) + fdt_sw.end_node() fdt_sw.end_node() - # 6. device-add (if devices added) - if devices_to_add: - fdt_sw.begin_node("device-add") - fdt_sw.property_string("mk,instance", instance_name) - - for idx, pci_id in enumerate(devices_to_add): - fdt_sw.begin_node(f"pci@{idx}") - fdt_sw.property_string("pci-id", pci_id) + if diff.memory_to_pool: + fdt_sw.begin_node("memory-add") + for idx, (node, size) in enumerate(diff.memory_to_pool): + fdt_sw.begin_node(f"memory@{idx}") + fdt_sw.property_u64("size", size) + if node >= 0: + fdt_sw.property_u32("numa-node-id", node) fdt_sw.end_node() - fdt_sw.end_node() + self._cpu_op(fdt_sw, "cpu-remove", diff.cpus_to_host) + self._cpu_op(fdt_sw, "cpu-add", diff.cpus_to_pool) + self._device_op(fdt_sw, "device-remove", diff.devices_to_host) + self._device_op(fdt_sw, "device-add", diff.devices_to_pool) + fdt_sw.end_node() # End __overlay__ fdt_sw.end_node() # End fragment@0 - fdt_sw.end_node() # End root dtb = fdt_sw.as_fdt() dtb.pack() return dtb.as_bytearray() - def _add_memory_operation(self, fdt_sw, fragment_id, operation, instance_name, base, size): - """Helper to add memory operation fragment.""" - import struct - - fdt_sw.begin_node(f"fragment@{fragment_id}") - fdt_sw.begin_node("__overlay__") - fdt_sw.begin_node(operation) - fdt_sw.property_string("mk,instance", instance_name) - - fdt_sw.begin_node("region@0") - reg_data = struct.pack(">QQ", base, size) - fdt_sw.property("reg", reg_data) + def _memory_item(self, fdt_sw, region, numa_node=None): + """Write a memory@0 item naming an existing range.""" + base, size = region + fdt_sw.begin_node("memory@0") + fdt_sw.property("reg", struct.pack(">QQ", base, size)) + if numa_node is not None: + fdt_sw.property_u32("numa-node-id", numa_node) fdt_sw.end_node() - fdt_sw.end_node() - fdt_sw.end_node() - fdt_sw.end_node() - - return fragment_id + 1 + def _cpu_op(self, fdt_sw, operation, cpu_ids, numa_node=None): + """Write a CPU operation node, or nothing when there are no CPUs.""" + if not cpu_ids: + return - def _add_cpu_operation( - self, fdt_sw, fragment_id, operation, instance_name, cpu_ids, numa_nodes - ): - """Helper to add CPU operation fragment.""" - fdt_sw.begin_node(f"fragment@{fragment_id}") - fdt_sw.begin_node("__overlay__") fdt_sw.begin_node(operation) - fdt_sw.property_string("mk,instance", instance_name) - for cpu_id in cpu_ids: fdt_sw.begin_node(f"cpu@{cpu_id}") fdt_sw.property("reg", pack_cpu_id(cpu_id)) + if numa_node is not None: + fdt_sw.property_u32("numa-node-id", numa_node) + fdt_sw.end_node() + fdt_sw.end_node() - if operation == "cpu-add" and numa_nodes: - fdt_sw.property_u32("numa-node", numa_nodes[0]) + def _device_op(self, fdt_sw, operation, pci_ids): + """Write a device operation node, or nothing when there are no devices.""" + if not pci_ids: + return + fdt_sw.begin_node(operation) + for idx, pci_id in enumerate(pci_ids): + fdt_sw.begin_node(f"pci@{idx}") + fdt_sw.property_string("pci-id", pci_id) fdt_sw.end_node() - - fdt_sw.end_node() fdt_sw.end_node() - fdt_sw.end_node() - - return fragment_id + 1 def _create_overlay_dtb( self, instances_to_add: dict, instances_to_update: dict, instances_to_remove: Set[str] @@ -299,6 +281,7 @@ def _create_overlay_dtb( all_instances = {**instances_to_add, **instances_to_update} for name, instance in all_instances.items(): fdt_sw.begin_node(f"fragment@{fragment_id}") + fdt_sw.property_string("target-path", self.INSTANCES_PATH) fdt_sw.begin_node("__overlay__") fdt_sw.begin_node("instance-create") @@ -319,8 +302,6 @@ def _create_overlay_dtb( fdt_sw.property("device-names", stringlist_data) if instance.resources.numa_nodes: - import struct - numa_data = struct.pack( ">" + "I" * len(instance.resources.numa_nodes), *instance.resources.numa_nodes ) @@ -363,6 +344,7 @@ def _create_overlay_dtb( for name in instances_to_remove: fdt_sw.begin_node(f"fragment@{fragment_id}") + fdt_sw.property_string("target-path", self.INSTANCES_PATH) fdt_sw.begin_node("__overlay__") fdt_sw.begin_node("instance-remove") fdt_sw.property_string("instance-name", name) diff --git a/tests/test_pool_overlay.py b/tests/test_pool_overlay.py new file mode 100644 index 0000000..0356a92 --- /dev/null +++ b/tests/test_pool_overlay.py @@ -0,0 +1,151 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pool transaction overlays and target-path addressing of instance overlays.""" + +import copy +import struct + +import libfdt + +from kerf.dtc.overlay import OverlayGenerator +from kerf.models import GlobalDeviceTree, PoolMemoryRegion +from kerf.pool_diff import PoolDiff + +GB = 1 << 30 +MISSING = -libfdt.FDT_ERR_NOTFOUND + + +def _ov(dtbo): + fdt = libfdt.Fdt(dtbo) + return fdt, fdt.path_offset("/fragment@0/__overlay__") + + +def _walk(fdt, offset=0): + yield offset + child = fdt.first_subnode(offset, quiet=[libfdt.FDT_ERR_NOTFOUND]) + while child >= 0: + for descendant in _walk(fdt, child): + yield descendant + child = fdt.next_subnode(child, quiet=[libfdt.FDT_ERR_NOTFOUND]) + + +def _grown(instance, cpus, memory_bytes, numa_nodes=None): + new = copy.deepcopy(instance) + new.resources.cpus = cpus + new.resources.memory_bytes = memory_bytes + new.resources.numa_nodes = numa_nodes + return new + + +def test_pool_overlay_layout(): + diff = PoolDiff( + cpus_to_pool=[8, 9], cpus_to_host=[4], + devices_to_pool=["0000:04:00.0"], devices_to_host=["0000:03:00.0"], + memory_to_pool=[(1, GB), (-1, GB // 2)], + memory_to_host=[PoolMemoryRegion(0x2_0000_0000, GB, 0)], + ) + fdt, ov = _ov(OverlayGenerator().generate_pool_overlay(diff)) + + frag = fdt.path_offset("/fragment@0") + assert fdt.getprop(frag, "target-path").as_str() == "/resources" + grow = fdt.subnode_offset(ov, "memory-add") + m0 = fdt.subnode_offset(grow, "memory@0") + assert fdt.getprop(m0, "size").as_uint64() == GB + assert fdt.getprop(m0, "numa-node-id").as_uint32() == 1 + m1 = fdt.subnode_offset(grow, "memory@1") + assert fdt.getprop(m1, "size").as_uint64() == GB // 2 + assert fdt.getprop(m1, "numa-node-id", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + + shrink = fdt.subnode_offset(ov, "memory-remove") + reg = bytes(fdt.getprop(fdt.subnode_offset(shrink, "memory@0"), "reg")) + assert struct.unpack(">QQ", reg) == (0x2_0000_0000, GB) + + assert fdt.getprop(fdt.subnode_offset(fdt.subnode_offset(ov, "cpu-add"), "cpu@8"), + "reg").as_uint64() == 8 + assert fdt.getprop(fdt.subnode_offset(fdt.subnode_offset(ov, "cpu-remove"), "cpu@4"), + "reg").as_uint64() == 4 + assert fdt.getprop(fdt.subnode_offset(fdt.subnode_offset(ov, "device-add"), "pci@0"), + "pci-id").as_str() == "0000:04:00.0" + assert fdt.getprop(fdt.subnode_offset(fdt.subnode_offset(ov, "device-remove"), "pci@0"), + "pci-id").as_str() == "0000:03:00.0" + + +def test_pool_overlay_omits_empty_ops(): + fdt, ov = _ov(OverlayGenerator().generate_pool_overlay(PoolDiff(cpus_to_pool=[5]))) + assert fdt.subnode_offset(ov, "memory-remove", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + assert fdt.subnode_offset(ov, "cpu-add") >= 0 + + +def test_update_overlay_targets_the_instance_path(sample_instances): + old = sample_instances["database"] + new = _grown(old, old.resources.cpus + [16], old.resources.memory_bytes + GB) + fdt, ov = _ov(OverlayGenerator().generate_update_overlay("database", old, new)) + + assert fdt.getprop(fdt.path_offset("/fragment@0"), "target-path").as_str() == "/instances/database" + for offset in _walk(fdt): + assert fdt.getprop(offset, "mk,instance", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + + add = fdt.subnode_offset(ov, "memory-add") + reg = bytes(fdt.getprop(fdt.subnode_offset(add, "memory@0"), "reg")) + assert struct.unpack(">QQ", reg) == (old.resources.memory_base + old.resources.memory_bytes, GB) + assert fdt.subnode_offset(add, "region@0", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + + +def test_update_overlay_pins_new_resources_to_the_instance_node(sample_instances): + old = sample_instances["database"] + new = _grown(old, old.resources.cpus + [16], old.resources.memory_bytes + GB, numa_nodes=[1]) + fdt, ov = _ov(OverlayGenerator().generate_update_overlay("database", old, new)) + + cpu = fdt.subnode_offset(fdt.subnode_offset(ov, "cpu-add"), "cpu@16") + assert fdt.getprop(cpu, "numa-node-id").as_uint32() == 1 + assert fdt.getprop(cpu, "numa-node", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + + mem = fdt.subnode_offset(fdt.subnode_offset(ov, "memory-add"), "memory@0") + assert fdt.getprop(mem, "numa-node-id").as_uint32() == 1 + + +def test_update_overlay_shrink_names_memory_items(sample_instances): + old = sample_instances["database"] + new = _grown(old, old.resources.cpus[:-1], old.resources.memory_bytes - GB) + fdt, ov = _ov(OverlayGenerator().generate_update_overlay("database", old, new)) + + remove = fdt.subnode_offset(ov, "memory-remove") + reg = bytes(fdt.getprop(fdt.subnode_offset(remove, "memory@0"), "reg")) + assert struct.unpack(">QQ", reg) == ( + old.resources.memory_base + new.resources.memory_bytes, GB) + + +def test_create_overlay_targets_the_instance_namespace(sample_hardware, sample_instances): + current = GlobalDeviceTree(hardware=sample_hardware, instances={}, device_references={}) + modified = GlobalDeviceTree( + hardware=sample_hardware, + instances={"database": sample_instances["database"]}, + device_references={}, + ) + fdt, ov = _ov(OverlayGenerator().generate_overlay(current, modified)) + + assert fdt.getprop(fdt.path_offset("/fragment@0"), "target-path").as_str() == "/instances" + create = fdt.subnode_offset(ov, "instance-create") + assert fdt.getprop(create, "instance-name").as_str() == "database" + for offset in _walk(fdt): + assert fdt.getprop(offset, "mk,instance", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + + +def test_removal_overlay_targets_the_instance_namespace(): + fdt, ov = _ov(OverlayGenerator().generate_removal_overlay("database")) + + assert fdt.getprop(fdt.path_offset("/fragment@0"), "target-path").as_str() == "/instances" + remove = fdt.subnode_offset(ov, "instance-remove") + assert fdt.getprop(remove, "instance-name").as_str() == "database" From b1c6ab7762bb51d282989f09fd72e224b2079e85 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 11:58:59 -0700 Subject: [PATCH 08/23] runtime: Share one overlay write path and read every pool chunk The write-and-check-the-transaction sequence lived in three copies, and the iomem helpers still assumed a single pool line although the kernel now registers one line per chunk. Collapse the copies into DeviceTreeManager.apply_dtbo(), report every chunk and note which ones still hold an allocation, and emit fragment unit-addresses in hex because the kernel parses them base 16. Signed-off-by: Cong Wang --- src/kerf/dtc/overlay.py | 4 +- src/kerf/resources.py | 67 +++++++++++++----- src/kerf/runtime.py | 137 ++++++++++++++----------------------- src/kerf/update/main.py | 41 +---------- tests/test_pool_overlay.py | 3 + tests/test_resources.py | 37 ++++++++++ 6 files changed, 145 insertions(+), 144 deletions(-) diff --git a/src/kerf/dtc/overlay.py b/src/kerf/dtc/overlay.py index 44d22e8..4135556 100644 --- a/src/kerf/dtc/overlay.py +++ b/src/kerf/dtc/overlay.py @@ -280,7 +280,7 @@ def _create_overlay_dtb( all_instances = {**instances_to_add, **instances_to_update} for name, instance in all_instances.items(): - fdt_sw.begin_node(f"fragment@{fragment_id}") + fdt_sw.begin_node(f"fragment@{fragment_id:x}") fdt_sw.property_string("target-path", self.INSTANCES_PATH) fdt_sw.begin_node("__overlay__") fdt_sw.begin_node("instance-create") @@ -343,7 +343,7 @@ def _create_overlay_dtb( fragment_id += 1 for name in instances_to_remove: - fdt_sw.begin_node(f"fragment@{fragment_id}") + fdt_sw.begin_node(f"fragment@{fragment_id:x}") fdt_sw.property_string("target-path", self.INSTANCES_PATH) fdt_sw.begin_node("__overlay__") fdt_sw.begin_node("instance-remove") diff --git a/src/kerf/resources.py b/src/kerf/resources.py index 8cb7461..bb19d81 100644 --- a/src/kerf/resources.py +++ b/src/kerf/resources.py @@ -24,12 +24,14 @@ from pathlib import Path from typing import List, Set, Optional, Tuple -from .lazy_cma import MULTIKERNEL_POOL_NAME from .models import GlobalDeviceTree from .exceptions import ResourceError IOMEM_PATH = "/proc/iomem" +#: iomem resource name the kernel registers every pool chunk under. +MULTIKERNEL_POOL_NAME = "Multikernel Memory Pool" + _IOMEM_RANGE_RE = re.compile(r"([0-9a-fA-F]+)-([0-9a-fA-F]+)\s*:\s*(.*)") @@ -52,43 +54,76 @@ def _parse_iomem_regions(iomem_path: str) -> List[Tuple[int, int, str]]: return regions -def get_memory_pool_from_iomem(iomem_path: str = IOMEM_PATH) -> Optional[Tuple[int, int]]: +def get_pool_chunks_from_iomem(iomem_path: str = IOMEM_PATH) -> List[Tuple[int, int]]: """ - Get the multikernel memory pool region from /proc/iomem. + List every multikernel pool chunk registered in /proc/iomem. Returns: - (base_address, size_bytes) or None if the pool is not registered + (base_address, size_bytes) tuples in /proc/iomem order + """ + return [ + (base, end - base + 1) + for base, end, name in _parse_iomem_regions(iomem_path) + if MULTIKERNEL_POOL_NAME in name + ] + + +def get_busy_chunks_from_iomem(iomem_path: str = IOMEM_PATH) -> Set[int]: + """ + Find the pool chunks that still hold an allocation. + + A chunk with any nested region cannot be returned to the host, so the + pool diff avoids picking it when it has a choice. + + Returns: + Base addresses of the chunks with at least one child region """ + chunks = get_pool_chunks_from_iomem(iomem_path) + busy = set() for base, end, name in _parse_iomem_regions(iomem_path): if MULTIKERNEL_POOL_NAME in name: - return (base, end - base + 1) - return None + continue + for chunk_base, chunk_size in chunks: + if chunk_base <= base and end <= chunk_base + chunk_size - 1: + busy.add(chunk_base) + return busy + + +def get_memory_pool_from_iomem(iomem_path: str = IOMEM_PATH) -> Optional[Tuple[int, int]]: + """ + Get the first multikernel memory pool chunk from /proc/iomem. + + Returns: + (base_address, size_bytes) or None if the pool is not registered + """ + chunks = get_pool_chunks_from_iomem(iomem_path) + return chunks[0] if chunks else None def get_pool_allocated_bytes(iomem_path: str = IOMEM_PATH) -> Optional[Tuple[int, int, int]]: """ Compute pool usage from /proc/iomem, the single source of truth. - Every region nested inside the pool range (instance memory, daxfs + Every region nested inside a pool chunk (instance memory, daxfs heaps, ...) counts as allocated. Overlapping and nested child regions are merged so nothing is double counted. Returns: - (pool_base, pool_bytes, allocated_bytes), or None if the pool is - not registered in /proc/iomem + (first_chunk_base, pool_bytes, allocated_bytes), or None if the + pool is not registered in /proc/iomem """ - pool = get_memory_pool_from_iomem(iomem_path) - if pool is None: + chunks = get_pool_chunks_from_iomem(iomem_path) + if not chunks: return None - pool_base, pool_bytes = pool - pool_end = pool_base + pool_bytes - 1 children = [] for base, end, name in _parse_iomem_regions(iomem_path): if MULTIKERNEL_POOL_NAME in name: continue - if base >= pool_base and end <= pool_end: - children.append((base, end)) + for chunk_base, chunk_size in chunks: + if base >= chunk_base and end <= chunk_base + chunk_size - 1: + children.append((base, end)) + break allocated = 0 current_base = current_end = None @@ -103,7 +138,7 @@ def get_pool_allocated_bytes(iomem_path: str = IOMEM_PATH) -> Optional[Tuple[int if current_base is not None: allocated += current_end - current_base + 1 - return (pool_base, pool_bytes, allocated) + return (chunks[0][0], sum(size for _, size in chunks), allocated) def get_available_cpus(tree: GlobalDeviceTree) -> Set[int]: diff --git a/src/kerf/runtime.py b/src/kerf/runtime.py index adcaf7c..081709a 100644 --- a/src/kerf/runtime.py +++ b/src/kerf/runtime.py @@ -173,6 +173,55 @@ def read_baseline(self) -> GlobalDeviceTree: """ return self.baseline_mgr.read_baseline() + def apply_dtbo(self, dtbo_data: bytes) -> str: + """ + Write a compiled overlay to the kernel and check the transaction. + + Args: + dtbo_data: DTBO blob to hand to /sys/fs/multikernel/overlays/new + + Returns: + Transaction ID (from kernel-created directory) + + Raises: + KernelInterfaceError: If the write fails or the kernel reports a + status other than applied + """ + if not self.overlays_new.exists(): + raise KernelInterfaceError(f"Overlay interface not found: {self.overlays_new}") + + try: + with open(self.overlays_new, "wb") as f: + f.write(dtbo_data) + except OSError as e: + raise KernelInterfaceError( + f"Failed to write overlay to {self.overlays_new}: {e}" + ) from e + + tx_id = self._find_latest_transaction() + if not tx_id: + raise KernelInterfaceError( + "Overlay written but kernel did not create transaction directory" + ) + + tx_dir = self.overlays_dir / f"tx_{tx_id}" + try: + status = (tx_dir / "status").read_text(encoding="utf-8").strip() + except OSError: + # An unreadable status may just mean the kernel is still processing. + return tx_id + + if status not in ("applied", "success", "ok"): + error_msg = f"Overlay transaction {tx_id} failed with status: '{status}'" + try: + instance_name = (tx_dir / "instance").read_text(encoding="utf-8").strip() + error_msg += f" (instance: {instance_name})" + except OSError: + pass + raise KernelInterfaceError(error_msg) + + return tx_id + def apply_overlay(self, current: GlobalDeviceTree, modified: GlobalDeviceTree) -> str: """ Apply overlay by writing DTBO to kernel. @@ -215,50 +264,7 @@ def apply_overlay(self, current: GlobalDeviceTree, modified: GlobalDeviceTree) - except Exception as e: raise KernelInterfaceError(f"Failed to generate overlay: {e}") from e - try: - if not self.overlays_new.exists(): - raise KernelInterfaceError(f"Overlay interface not found: {self.overlays_new}") - - with open(self.overlays_new, "wb") as f: - f.write(dtbo_data) - - tx_id = self._find_latest_transaction() - if not tx_id: - raise KernelInterfaceError( - "Overlay written but kernel did not create transaction directory" - ) - - # Verify transaction succeeded by checking status - tx_dir = self.overlays_dir / f"tx_{tx_id}" - status_file = tx_dir / "status" - - if status_file.exists(): - try: - with open(status_file, "r", encoding="utf-8") as f: - status = f.read().strip() - if status not in ("applied", "success", "ok"): - error_msg = f"Overlay transaction {tx_id} failed with status: '{status}'" - instance_file = tx_dir / "instance" - if instance_file.exists(): - try: - with open(instance_file, "r", encoding="utf-8") as f: - instance_name = f.read().strip() - error_msg += f" (instance: {instance_name})" - except OSError: - pass - - raise KernelInterfaceError(error_msg) - except OSError: - # If we can't read status, assume it might still be processing - # But warn that we couldn't verify - pass - - return tx_id - - except OSError as e: - raise KernelInterfaceError( - f"Failed to write overlay to {self.overlays_new}: {e}" - ) from e + return self.apply_dtbo(dtbo_data) def apply_removal_overlay(self, instance_name: str) -> str: """ @@ -284,48 +290,7 @@ def apply_removal_overlay(self, instance_name: str) -> str: except Exception as e: raise KernelInterfaceError(f"Failed to generate removal overlay: {e}") from e - try: - if not self.overlays_new.exists(): - raise KernelInterfaceError(f"Overlay interface not found: {self.overlays_new}") - - with open(self.overlays_new, "wb") as f: - f.write(dtbo_data) - - tx_id = self._find_latest_transaction() - if not tx_id: - raise KernelInterfaceError( - "Overlay written but kernel did not create transaction directory" - ) - - tx_dir = self.overlays_dir / f"tx_{tx_id}" - status_file = tx_dir / "status" - - if status_file.exists(): - try: - with open(status_file, "r", encoding="utf-8") as f: - status = f.read().strip() - if status not in ("applied", "success", "ok"): - error_msg = ( - f"Overlay transaction {tx_id} failed with status: '{status}'" - ) - instance_file = tx_dir / "instance" - if instance_file.exists(): - try: - with open(instance_file, "r", encoding="utf-8") as f: - tx_instance_name = f.read().strip() - error_msg += f" (instance: {tx_instance_name})" - except OSError: - pass - raise KernelInterfaceError(error_msg) - except OSError: - pass - - return tx_id - - except OSError as e: - raise KernelInterfaceError( - f"Failed to write overlay to {self.overlays_new}: {e}" - ) from e + return self.apply_dtbo(dtbo_data) def _find_latest_transaction(self) -> Optional[str]: """Find the latest transaction ID from kernel-created directories.""" diff --git a/src/kerf/update/main.py b/src/kerf/update/main.py index 64ba0b1..c51a013 100644 --- a/src/kerf/update/main.py +++ b/src/kerf/update/main.py @@ -368,46 +368,7 @@ def apply_update_operation(current): current = manager.read_baseline() dtbo_data = apply_update_operation(current) - try: - if not manager.overlays_new.exists(): - raise KernelInterfaceError( - f"Overlay interface not found: {manager.overlays_new}" - ) - - with open(manager.overlays_new, 'wb') as f: - f.write(dtbo_data) - - tx_id = manager._find_latest_transaction() # pylint: disable=protected-access - if not tx_id: - raise KernelInterfaceError( - "Overlay written but kernel did not create transaction directory" - ) - - tx_dir = manager.overlays_dir / f"tx_{tx_id}" - status_file = tx_dir / "status" - - if status_file.exists(): - try: - with open(status_file, 'r', encoding='utf-8') as f: - status = f.read().strip() - if status not in ("applied", "success", "ok"): - error_msg = f"Overlay transaction {tx_id} failed with status: '{status}'" - instance_file = tx_dir / "instance" - if instance_file.exists(): - try: - with open(instance_file, 'r', encoding='utf-8') as f: - instance_name_from_tx = f.read().strip() - error_msg += f" (instance: {instance_name_from_tx})" - except OSError: - pass - raise KernelInterfaceError(error_msg) - except OSError: - pass - - except OSError as e: - raise KernelInterfaceError( - f"Failed to write overlay to {manager.overlays_new}: {e}" - ) from e + tx_id = manager.apply_dtbo(dtbo_data) click.echo(f"✓ Updated instance '{name}' (transaction {tx_id})") if verbose: diff --git a/tests/test_pool_overlay.py b/tests/test_pool_overlay.py index 0356a92..6f397dc 100644 --- a/tests/test_pool_overlay.py +++ b/tests/test_pool_overlay.py @@ -67,6 +67,9 @@ def test_pool_overlay_layout(): m1 = fdt.subnode_offset(grow, "memory@1") assert fdt.getprop(m1, "size").as_uint64() == GB // 2 assert fdt.getprop(m1, "numa-node-id", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + # The kernel picks the chunk, so a grow item never names an address. + assert fdt.getprop(m0, "reg", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + assert fdt.getprop(m1, "reg", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING shrink = fdt.subnode_offset(ov, "memory-remove") reg = bytes(fdt.getprop(fdt.subnode_offset(shrink, "memory@0"), "reg")) diff --git a/tests/test_resources.py b/tests/test_resources.py index 09112d4..6efb193 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -24,6 +24,8 @@ get_allocated_memory_regions_from_iomem, get_memory_pool_from_iomem, get_pool_allocated_bytes, + get_pool_chunks_from_iomem, + get_busy_chunks_from_iomem, find_available_memory_base, validate_cpu_allocation, validate_memory_allocation, @@ -303,3 +305,38 @@ def test_get_pool_allocated_bytes_merges_nested_regions(self, tmp_path): ) usage = get_pool_allocated_bytes(str(path)) assert usage == (0x40000000, 0x40000000, 0x8000000) + + TWO_CHUNKS = "\n".join( + [ + "100000000-13fffffff : Multikernel Memory Pool", + " 100000000-10fffffff : mk-instance-1-web-region-0", + "200000000-21fffffff : Multikernel Memory Pool", + ] + ) + + @pytest.fixture + def two_chunk_iomem(self, tmp_path): + path = tmp_path / "iomem" + path.write_text(self.TWO_CHUNKS + "\n", encoding="utf-8") + return str(path) + + def test_pool_chunks_and_busy(self, two_chunk_iomem): + assert get_pool_chunks_from_iomem(two_chunk_iomem) == [ + (0x100000000, 1 << 30), + (0x200000000, 1 << 29), + ] + assert get_busy_chunks_from_iomem(two_chunk_iomem) == {0x100000000} + + def test_first_chunk_and_allocation_across_chunks(self, two_chunk_iomem): + assert get_memory_pool_from_iomem(two_chunk_iomem) == (0x100000000, 1 << 30) + assert get_pool_allocated_bytes(two_chunk_iomem) == ( + 0x100000000, + (1 << 30) + (1 << 29), + 1 << 28, + ) + + def test_no_chunks(self, tmp_path): + path = tmp_path / "iomem" + path.write_text("00001000-0009ffff : System RAM\n", encoding="utf-8") + assert get_pool_chunks_from_iomem(str(path)) == [] + assert get_busy_chunks_from_iomem(str(path)) == set() From f9b8d790f998f84210f9b819201886769cedb937 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 11:58:59 -0700 Subject: [PATCH 09/23] init: Reconcile the live pool against the requested baseline The kernel refuses a baseline write once the pool holds resources, so re-running kerf init failed with EBUSY. Read the pool back, diff it against the request and apply the difference as a /resources overlay transaction, leaving the baseline write for an empty pool only. The --memory option now describes the request rather than a runtime allocation, so the lazy_cma client is gone, and --teardown hands every pool resource back to the host. Signed-off-by: Cong Wang --- README.md | 12 +- src/kerf/init/main.py | 402 +++++++++++++++++++++++---------- src/kerf/lazy_cma.py | 132 ----------- src/kerf/show/main.py | 6 +- tests/test_init_memory_spec.py | 41 ++++ tests/test_init_reconcile.py | 152 +++++++++++++ 6 files changed, 482 insertions(+), 263 deletions(-) delete mode 100644 src/kerf/lazy_cma.py create mode 100644 tests/test_init_memory_spec.py create mode 100644 tests/test_init_reconcile.py diff --git a/README.md b/README.md index dce2d7f..9dc41d1 100644 --- a/README.md +++ b/README.md @@ -127,11 +127,15 @@ Baseline DTB (static) ### Command Line Interface ```bash -# Initialize resource pool with CPUs (memory parsed from /proc/iomem) -kerf init --cpus=4-7 +# Initialize resource pool with CPUs and pool memory +kerf init --cpus=4-7 --memory=2GB -# Initialize with CPUs and devices -kerf init --cpus=4-31 --devices=enp9s0_dev,nvme0 +# Initialize with CPUs, per-node memory and devices +kerf init --cpus=4-31 --memory=node0:8GB,node1:8GB --devices=enp9s0_dev,nvme0 + +# Re-run to reshape the live pool, or hand everything back to the host +kerf init --cpus=4-15 --memory=4GB +kerf init --teardown # Create kernel instance with resource allocation kerf create web-server --cpus=4-7 --memory=2GB diff --git a/src/kerf/init/main.py b/src/kerf/init/main.py index cb819e8..4f25979 100644 --- a/src/kerf/init/main.py +++ b/src/kerf/init/main.py @@ -25,7 +25,7 @@ import re import sys from pathlib import Path -from typing import Optional +from typing import Dict, Optional import click import libfdt @@ -38,8 +38,7 @@ from ..baseline import BaselineManager from ..create.main import parse_cpu_spec, parse_device_list, parse_memory_spec from ..dtc.parser import DeviceTreeParser -from ..lazy_cma import LAZY_CMA_DEVICE, allocate_multikernel_pool -from ..resources import get_memory_pool_from_iomem +from ..resources import get_busy_chunks_from_iomem from ..dtc.reporter import ValidationReporter from ..dtc.validator import MultikernelValidator from ..exceptions import KernelInterfaceError, ParseError, ValidationError @@ -49,8 +48,9 @@ GlobalDeviceTree, HardwareInventory, MemoryAllocation, - PoolMemoryRegion, ) +from ..pool_diff import PoolDiff, compute_pool_diff +from ..runtime import DeviceTreeManager MULTIKERNEL_MOUNT_POINT = "/sys/fs/multikernel" @@ -354,6 +354,56 @@ def get_valid_apic_ids_from_system() -> Optional[set]: return None +_NODE_SPEC = re.compile(r"^node(\d+):(.+)$") + +PAGE_SIZE = 4096 + + +def parse_memory_request(spec: str) -> Dict[int, int]: + """ + Parse a pool memory request into per-NUMA-node sizes. + + "2GB" asks for 2GB on any node (node -1), "node0:8GB,node1:8GB" asks + for a specific amount per node. The two forms cannot be mixed. + + Args: + spec: Memory specification string + + Returns: + Mapping of NUMA node id (-1 for any node) to size in bytes + + Raises: + ValueError: If the specification is malformed or a size is zero + or not page aligned + """ + parts = [p.strip() for p in spec.split(",") if p.strip()] + if not parts: + raise ValueError("empty memory specification") + + requested: Dict[int, int] = {} + for part in parts: + match = _NODE_SPEC.match(part) + if match: + node, size = int(match.group(1)), parse_memory_spec(match.group(2)) + elif part.lower().startswith("node"): + raise ValueError(f"invalid NUMA node spec '{part}' (expected nodeN:SIZE)") + else: + node, size = -1, parse_memory_spec(part) + if size <= 0: + raise ValueError(f"memory size in '{part}' must be greater than zero") + if size % PAGE_SIZE: + raise ValueError( + f"memory size in '{part}' must be a multiple of {PAGE_SIZE} bytes" + ) + if node in requested: + raise ValueError(f"node {node} specified twice") + requested[node] = size + + if -1 in requested and len(requested) > 1: + raise ValueError("cannot mix a plain size with nodeN: sizes") + return requested + + def build_baseline_from_cmdline( cpus: str, memory: Optional[str] = None, @@ -365,9 +415,8 @@ def build_baseline_from_cmdline( Args: cpus: CPU specification string (e.g., "4-7" or "4,5,6,7") - memory: Optional pool size to allocate at runtime via /dev/lazy_cma - (e.g., "1GB"); when omitted an existing pool must already - be registered in /proc/iomem + memory: Pool memory request, either "2GB" for any node or + "node0:8GB,node1:8GB" for specific nodes devices: Optional device names (comma-separated, e.g., "enp9s0_dev,nvme0") verbose: Whether to print verbose output @@ -375,14 +424,18 @@ def build_baseline_from_cmdline( GlobalDeviceTree with resources only (no instances) Raises: - ValueError: If CPU or memory specification is invalid - KernelInterfaceError: If the memory pool cannot be found or allocated + ValueError: If the CPU or memory specification is invalid + KernelInterfaceError: If the system topology cannot be read """ try: cpu_list = parse_cpu_spec(cpus) except ValueError as e: raise ValueError(f"Invalid CPU specification '{cpus}': {e}") from e + if not memory: + raise ValueError("--memory is required") + requested = parse_memory_request(memory) + # Validate against valid APIC IDs on the system valid_apic_ids = get_valid_apic_ids_from_system() if valid_apic_ids is None: @@ -411,45 +464,16 @@ def build_baseline_from_cmdline( host_reserved_cpus = [0] cpu_list = sorted(list(available_cpus)) - memory_pool = get_memory_pool_from_iomem() - if memory_pool is None: - if not memory: - raise KernelInterfaceError( - "Could not find multikernel memory pool in /proc/iomem. " - "Pass --memory=SIZE (e.g. --memory=1GB) to allocate the pool " - f"at runtime via {LAZY_CMA_DEVICE}." - ) - - pool_bytes = parse_memory_spec(memory) - pool_base = allocate_multikernel_pool(pool_bytes) - if verbose: - click.echo(f"Allocated multikernel memory pool via {LAZY_CMA_DEVICE}:") - click.echo(f" Base: {hex(pool_base)}") - click.echo(f" Size: {pool_bytes} bytes ({pool_bytes / (1024**3):.2f} GB)") - - # Re-read /proc/iomem so the baseline reflects exactly what the - # kernel registered for the allocation. - memory_pool = get_memory_pool_from_iomem() or (pool_base, pool_bytes) - elif memory: - click.echo( - "Note: multikernel memory pool already exists in /proc/iomem; " - "ignoring --memory", - err=True, - ) - memory_pool_base, memory_pool_bytes = memory_pool - - total_bytes = memory_pool_base + memory_pool_bytes - host_reserved_bytes = memory_pool_base + total_bytes = sum(requested.values()) if verbose: click.echo(f"Parsed APIC ID specification: {cpus}") click.echo(f" Valid APIC IDs on system: {sorted(valid_apic_ids)}") click.echo(f" Host-reserved APIC IDs: {host_reserved_cpus}") click.echo(f" Available APIC IDs: {cpu_list}") - click.echo("Memory pool from /proc/iomem:") - click.echo(f" Base: {hex(memory_pool_base)}") - click.echo(f" Size: {memory_pool_bytes} bytes ({memory_pool_bytes / (1024**3):.2f} GB)") - click.echo(f" Total bytes: {total_bytes} bytes ({total_bytes / (1024**3):.2f} GB)") - click.echo(f" Host-reserved: {host_reserved_bytes} bytes ({host_reserved_bytes / (1024**3):.2f} GB)") + click.echo("Requested pool memory:") + for node, size in sorted(requested.items()): + where = "any node" if node < 0 else f"node {node}" + click.echo(f" {where}: {size} bytes ({size / (1024**3):.2f} GB)") cpu_allocation = CPUAllocation( total=total_cpus, @@ -459,8 +483,8 @@ def build_baseline_from_cmdline( memory_allocation = MemoryAllocation( total_bytes=total_bytes, - host_reserved_bytes=host_reserved_bytes, - regions=[PoolMemoryRegion(base=memory_pool_base, size=memory_pool_bytes)] + host_reserved_bytes=0, + requested=requested ) device_dict = {} @@ -503,18 +527,148 @@ def build_baseline_from_cmdline( return tree +def build_teardown_tree() -> GlobalDeviceTree: + """Build the empty requested state: every pool resource goes back to the host.""" + apic_ids = get_valid_apic_ids_from_system() or set() + cpu_allocation = CPUAllocation( + total=(max(apic_ids) + 1) if apic_ids else 0, + host_reserved=sorted(apic_ids), + available=[] + ) + + hardware = HardwareInventory( + cpus=cpu_allocation, + memory=MemoryAllocation(total_bytes=0, host_reserved_bytes=0, requested={}), + devices={} + ) + + return GlobalDeviceTree(hardware=hardware, instances={}, device_references={}) + + +def pool_is_live(current: Optional[GlobalDeviceTree]) -> bool: + """Whether the kernel already holds pool resources we have to diff against.""" + if current is None: + return False + hardware = current.hardware + return bool(hardware.cpus.available or hardware.memory.regions or hardware.devices) + + +def _print_diff(diff: PoolDiff) -> None: + """Show what would move between the host and the pool.""" + def line(label, items): + if items: + click.echo(f" {label}: {', '.join(str(i) for i in items)}") + + line("CPUs to pool", diff.cpus_to_pool) + line("CPUs to host", diff.cpus_to_host) + line("Memory to pool", [ + f"{size >> 20} MB" + (f" on node {node}" if node >= 0 else "") + for node, size in diff.memory_to_pool + ]) + line("Memory to host", [f"{hex(r.base)} ({r.size >> 20} MB)" for r in diff.memory_to_host]) + line("Devices to pool", diff.devices_to_pool) + line("Devices to host", diff.devices_to_host) + + +def _report_shortfall(live: GlobalDeviceTree, requested: GlobalDeviceTree) -> None: + """Warn when the kernel could not shrink the pool all the way down.""" + for node, want in requested.hardware.memory.requested.items(): + if node < 0: + have = live.hardware.memory.memory_pool_bytes + else: + have = live.hardware.memory.bytes_on_node(node) + if have > want: + where = node if node >= 0 else "any" + click.echo( + f"Note: node {where} still holds {(have - want) >> 20} MB more than " + "requested; only whole idle chunks can be returned", + err=True, + ) + + +def reconcile_pool( + current: Optional[GlobalDeviceTree], + requested: GlobalDeviceTree, + busy_chunks: set, + dry_run: bool, + manager, + baseline_mgr, +) -> Optional[PoolDiff]: + """ + Bring the live pool in line with the requested baseline. + + An empty pool takes the baseline write; a live pool is reconciled with + a /resources overlay transaction, since the kernel refuses a baseline + write once it owns resources. + + Args: + current: Baseline read back from the kernel, or None if there is none + requested: Requested state + busy_chunks: Bases of pool chunks that still hold an allocation + dry_run: Report the plan without touching the kernel + manager: DeviceTreeManager used to apply the overlay + baseline_mgr: BaselineManager used for the initial write + + Returns: + The applied (or planned) difference, or None if a baseline was written + + Raises: + KernelInterfaceError: If the kernel rejects the write or the overlay + """ + if not pool_is_live(current): + if dry_run: + click.echo("Baseline validation passed; would write the initial baseline (dry-run)") + return None + baseline_mgr.write_baseline(requested) + click.echo("✓ Baseline applied to kernel successfully") + return None + + diff = compute_pool_diff(current, requested, busy_chunks=busy_chunks) + _print_diff(diff) + if diff.is_empty(): + click.echo("✓ Pool already matches the request; nothing to do") + return diff + if dry_run: + click.echo("Would apply the changes above (dry-run)") + return diff + + with manager._acquire_lock(): # pylint: disable=protected-access + tx_id = manager.apply_dtbo(manager.overlay_gen.generate_pool_overlay(diff)) + click.echo(f"✓ Pool updated (transaction {tx_id})") + _report_shortfall(baseline_mgr.read_baseline(), requested) + return diff + + +def _dump_baseline_dts(baseline_mgr: BaselineManager, tree: GlobalDeviceTree) -> None: + """Print the DTS the baseline write would carry.""" + try: + dtb_data = baseline_mgr.extractor.generate_global_dtb(tree) + fdt = libfdt.Fdt(dtb_data) + dts_parser = DeviceTreeParser() + dts_parser.fdt = fdt + dts_lines = dts_parser._fdt_to_dts_recursive(0, 0) # pylint: disable=protected-access + + click.echo("Debug: Baseline DTS source being written to kernel:") + click.echo("─" * 70) + click.echo('\n'.join(dts_lines)) + click.echo("─" * 70) + except Exception as e: # pylint: disable=broad-except + click.echo(f"Debug: Failed to convert baseline DTB to DTS: {e}", err=True) + + @click.command() @click.pass_context @click.option('--input', '-i', help='Input DTS or DTB file containing all resources. Mutually exclusive with --cpus, --memory and --devices. When used, all resources must come from the file.') -@click.option('--cpus', '-c', help='APIC ID specification for baseline (e.g., "128-134" or "128,130,132"). Use physical APIC IDs, not logical CPU numbers. Mutually exclusive with --input. Memory comes from --memory or an existing pool in /proc/iomem.') -@click.option('--memory', '-m', help='Memory pool size to allocate at runtime via /dev/lazy_cma (e.g., "1GB", "512MB"). If omitted, an existing pool is discovered from /proc/iomem. Mutually exclusive with --input.') +@click.option('--cpus', '-c', help='APIC ID specification for baseline (e.g., "128-134" or "128,130,132"). Use physical APIC IDs, not logical CPU numbers. Mutually exclusive with --input.') +@click.option('--memory', '-m', help='Pool memory: SIZE for any node (e.g. "2GB") or per-node "node0:8GB,node1:8GB". Required with --cpus, mutually exclusive with --input.') @click.option('--devices', '-d', help='Device names (comma-separated, e.g., "enp9s0_dev,nvme0"). Mutually exclusive with --input. Creates minimal device entries in baseline.') +@click.option('--teardown', is_flag=True, help='Return every pool resource to the host. Mutually exclusive with --input, --cpus, --memory and --devices.') @click.option('--dry-run', is_flag=True, help='Validate without applying') @click.option('--report', is_flag=True, help='Generate detailed validation report') @click.option('--format', type=click.Choice(['text', 'json', 'yaml']), default='text', help='Report format (default: text)') @click.option('--verbose', '-v', is_flag=True, help='Verbose output') -def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: Optional[str], devices: Optional[str], dry_run: bool, report: bool, format: str, verbose: bool): +def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: Optional[str], devices: Optional[str], teardown: bool, dry_run: bool, report: bool, format: str, verbose: bool): """ Initialize baseline device tree configuration. @@ -522,33 +676,41 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: available for allocation. The baseline must contain ONLY resources (no instances). Instances are created via 'kerf create' using overlays. - By default, the baseline is applied to the kernel after validation. - Use --dry-run to validate without applying. + The command is idempotent: the requested state is compared against the + pool the kernel reports, and only the difference is applied. An empty + pool takes the baseline write; a live pool is reconciled with a + /resources overlay transaction. Use --teardown to return everything to + the host, and --dry-run to see the plan without applying it. You can either provide a DTS/DTB file via --input, or construct the - baseline from command line arguments using --cpus. These options are - mutually exclusive - when using --input, all resources must come from - the DTS file. With --cpus, the memory pool is allocated at runtime - via /dev/lazy_cma when --memory is given, or discovered from - /proc/iomem otherwise. + request from command line arguments using --cpus and --memory. These + options are mutually exclusive. Examples: # Initialize from DTS file (all resources from file) kerf init --input=hardware.dts - # Initialize from command line, allocating a 1GB pool at runtime + # Request 1GB of pool memory on any node kerf init --cpus=128-134 --memory=1GB - # Initialize reusing a pool already registered in /proc/iomem - kerf init --cpus=128-134 + # Request memory per NUMA node + kerf init --cpus=128-134 --memory=node0:8GB,node1:8GB + + # Shrink the pool back to 1GB and 2 CPUs + kerf init --cpus=128,129 --memory=1GB - # Initialize with APIC IDs and devices - kerf init --cpus=128,130,132 --memory=1GB --devices=enp9s0_dev,nvme0 + # Return every pool resource to the host + kerf init --teardown - # Validate baseline without applying - kerf init --input=hardware.dts --dry-run + # Show what would change without applying + kerf init --cpus=128-134 --memory=1GB --dry-run """ try: + if teardown and (input or cpus or memory or devices): + click.echo("Error: --teardown cannot be combined with --input, --cpus, --memory or --devices.", err=True) + click.echo("--teardown returns every pool resource to the host.", err=True) + sys.exit(2) + # Validate that --input and resource specification options are mutually exclusive # When using --input, all resources must come from the DTS file if input and (cpus or memory or devices): @@ -564,18 +726,21 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: click.echo("Use either --input for a complete DTS/DTB file, or command-line options to construct baseline.", err=True) sys.exit(2) - if not input and not cpus: - click.echo("Error: Either --input or --cpus must be specified", err=True) + if not input and not cpus and not teardown: + click.echo("Error: Either --input, --cpus or --teardown must be specified", err=True) click.echo("\nUsage:", err=True) click.echo(" kerf init --input=hardware.dts", err=True) click.echo(" kerf init --cpus=4-7 --memory=1GB", err=True) - click.echo(" kerf init --cpus=4-7 --memory=1GB --devices=enp9s0_dev", err=True) + click.echo(" kerf init --cpus=4-7 --memory=node0:1GB --devices=enp9s0_dev", err=True) + click.echo(" kerf init --teardown", err=True) sys.exit(2) parser = DeviceTreeParser() dts_content = None - if input: + if teardown: + tree = build_teardown_tree() + elif input: # Parse from input file input_path = Path(input) @@ -616,74 +781,63 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: click.echo("\nInstances should be created via 'kerf create'", err=True) sys.exit(1) - validator = MultikernelValidator() - if dts_content is not None: - input_path_str = str(input) if input else "command-line" - validator.set_dts_context(dts_content, input_path_str) + # The teardown request holds no resources at all, which the resource + # validator reads as an unusable pool. + if not teardown: + validator = MultikernelValidator() + if dts_content is not None: + input_path_str = str(input) if input else "command-line" + validator.set_dts_context(dts_content, input_path_str) + + validation_result = validator.validate(tree) - validation_result = validator.validate(tree) + if report: + reporter = ValidationReporter() + report_text = reporter.generate_report(validation_result, tree, verbose, format) + click.echo(report_text) + if not validation_result.is_valid: + sys.exit(1) + return - if report: - reporter = ValidationReporter() - report_text = reporter.generate_report(validation_result, tree, verbose, format) - click.echo(report_text) if not validation_result.is_valid: + click.echo("Validation failed:", err=True) + for error in validation_result.errors: + click.echo(f" ✗ {error}", err=True) + if validation_result.warnings: + click.echo("\nWarnings:", err=True) + for warning in validation_result.warnings: + click.echo(f" ⚠ {warning}", err=True) sys.exit(1) - return - - if not validation_result.is_valid: - click.echo("Validation failed:", err=True) - for error in validation_result.errors: - click.echo(f" ✗ {error}", err=True) - if validation_result.warnings: - click.echo("\nWarnings:", err=True) - for warning in validation_result.warnings: - click.echo(f" ⚠ {warning}", err=True) - sys.exit(1) - if verbose: - click.echo("✓ Baseline validation passed") - if validation_result.warnings: - click.echo("\nWarnings:") - for warning in validation_result.warnings: - click.echo(f" ⚠ {warning}") + if verbose: + click.echo("✓ Baseline validation passed") + if validation_result.warnings: + click.echo("\nWarnings:") + for warning in validation_result.warnings: + click.echo(f" ⚠ {warning}") debug = ctx.obj.get('debug', False) if ctx and ctx.obj else False - if dry_run: - click.echo(" Baseline validation passed") - click.echo(" Baseline would be applied (dry-run mode)") - else: - try: - mount_multikernel_fs(verbose=verbose) + manager = DeviceTreeManager() + mount_multikernel_fs(verbose=verbose) - if debug: - try: - dtb_data = baseline_mgr.extractor.generate_global_dtb(tree) - fdt = libfdt.Fdt(dtb_data) - dts_parser = DeviceTreeParser() - dts_parser.fdt = fdt - dts_lines = dts_parser._fdt_to_dts_recursive(0, 0) # pylint: disable=protected-access - dts_content = '\n'.join(dts_lines) - - click.echo("Debug: Baseline DTS source being written to kernel:") - click.echo("─" * 70) - click.echo(dts_content) - click.echo("─" * 70) - except Exception as e: - click.echo(f"Debug: Failed to convert baseline DTB to DTS: {e}", err=True) + current = None + if baseline_mgr.baseline_path.exists() and baseline_mgr.baseline_path.stat().st_size: + current = baseline_mgr.read_baseline() - if verbose: - click.echo("Writing baseline to kernel...") - baseline_mgr.write_baseline(tree) - click.echo("✓ Baseline applied to kernel successfully") - click.echo(" Baseline: /sys/fs/multikernel/device_tree") - except KernelInterfaceError as e: - click.echo(f"Error: Failed to apply baseline: {e}", err=True) - if verbose: - import traceback - traceback.print_exc() - sys.exit(1) + if debug and not pool_is_live(current): + _dump_baseline_dts(baseline_mgr, tree) + + try: + reconcile_pool(current, tree, get_busy_chunks_from_iomem(), dry_run, + manager, baseline_mgr) + except KernelInterfaceError as e: + click.echo(f"Error: {e}", err=True) + click.echo("Run 'dmesg | tail' for the kernel's reason (it names the busy CPU or chunk).", err=True) + if verbose: + import traceback + traceback.print_exc() + sys.exit(1) except ParseError as e: click.echo(f"Error: Failed to parse input file: {e}", err=True) diff --git a/src/kerf/lazy_cma.py b/src/kerf/lazy_cma.py deleted file mode 100644 index b204433..0000000 --- a/src/kerf/lazy_cma.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2026 Multikernel Technologies, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Client for the lazy_cma runtime contiguous memory allocator. - -kerf allocates the multikernel memory pool at runtime through -/dev/lazy_cma instead of requiring a boot-time reservation. Allocating -under the canonical resource name ("Multikernel Memory Pool") makes the -pool discoverable in /proc/iomem, which is where kerf and the kernel -tooling look it up. -""" - -import fcntl -import os -import struct - -from .exceptions import KernelInterfaceError - -LAZY_CMA_DEVICE = "/dev/lazy_cma" - -#: iomem resource name the multikernel stack discovers the pool by. -MULTIKERNEL_POOL_NAME = "Multikernel Memory Pool" - -_LAZY_CMA_IOC_MAGIC = ord("H") -_LAZY_CMA_NAME_MAX = 64 - -# struct lazy_cma_allocation_data { -# __u64 len; -# __u64 phys_addr; /* out */ -# __s32 node; /* NUMA node; -1 = any */ -# __u32 pad; -# char name[64]; /* iomem resource name */ -# }; -_ALLOC_FORMAT = f" int: - """Encode an ioctl request number (matches the kernel's _IOC macro).""" - return (direction << 30) | (size << 16) | (_LAZY_CMA_IOC_MAGIC << 8) | nr - - -LAZY_CMA_IOCTL_ALLOC = _ioc(_IOC_READ | _IOC_WRITE, 0x0, _ALLOC_SIZE) - - -def allocate(size_bytes: int, name: str, node: int = -1) -> int: - """ - Allocate physically contiguous memory via /dev/lazy_cma. - - Args: - size_bytes: Allocation size in bytes (page granularity in the kernel) - name: iomem resource name for the allocation - node: NUMA node to allocate from, or -1 for any node - - Returns: - Physical base address of the allocation - - Raises: - KernelInterfaceError: If the device is unavailable or allocation fails - ValueError: If the arguments are invalid - """ - if size_bytes <= 0: - raise ValueError(f"Allocation size must be positive, got {size_bytes}") - - encoded_name = name.encode("utf-8") - if len(encoded_name) >= _LAZY_CMA_NAME_MAX: - raise ValueError( - f"Resource name too long ({len(encoded_name)} bytes, " - f"max {_LAZY_CMA_NAME_MAX - 1})" - ) - - request = bytearray( - struct.pack(_ALLOC_FORMAT, size_bytes, 0, node, 0, encoded_name) - ) - - try: - fd = os.open(LAZY_CMA_DEVICE, os.O_RDWR) - except FileNotFoundError as exc: - raise KernelInterfaceError( - f"{LAZY_CMA_DEVICE} not found. Load the lazy_cma kernel module " - "first (e.g. insmod lazy_cma.ko)." - ) from exc - except PermissionError as exc: - raise KernelInterfaceError( - f"Permission denied opening {LAZY_CMA_DEVICE}. kerf must run as " - "root to allocate pool memory." - ) from exc - - try: - fcntl.ioctl(fd, LAZY_CMA_IOCTL_ALLOC, request) - except OSError as exc: - raise KernelInterfaceError( - f"lazy_cma allocation of {size_bytes} bytes failed: " - f"{os.strerror(exc.errno)} (errno {exc.errno}). " - "The system may not have enough contiguous free memory; " - "try a smaller pool size." - ) from exc - finally: - os.close(fd) - - _, phys_addr, _, _, _ = struct.unpack(_ALLOC_FORMAT, bytes(request)) - if phys_addr == 0: - raise KernelInterfaceError( - f"lazy_cma returned physical address 0 for a {size_bytes} byte " - "allocation" - ) - - return phys_addr - - -def allocate_multikernel_pool(size_bytes: int, node: int = -1) -> int: - """ - Allocate the multikernel memory pool under its canonical iomem name. - - Returns the physical base address of the pool. - """ - return allocate(size_bytes, MULTIKERNEL_POOL_NAME, node) diff --git a/src/kerf/show/main.py b/src/kerf/show/main.py index 2b02c09..0fa515f 100644 --- a/src/kerf/show/main.py +++ b/src/kerf/show/main.py @@ -301,9 +301,9 @@ def display_baseline_info(tree: GlobalDeviceTree, verbose: bool = False): f" Host Reserved: {reserved_gb:.2f} GB ({hardware.memory.host_reserved_bytes} bytes)" ) - # /proc/iomem is the source of truth for the lazy_cma pool and the - # instance allocations carved out of it; the baseline tree only - # snapshots the pool at init time. + # /proc/iomem is the source of truth for the pool chunks and the + # instance allocations carved out of them; the baseline tree only + # snapshots the pool as of the last transaction. usage = get_pool_allocated_bytes() if usage is not None: pool_base, pool_bytes, allocated_bytes = usage diff --git a/tests/test_init_memory_spec.py b/tests/test_init_memory_spec.py new file mode 100644 index 0000000..5537732 --- /dev/null +++ b/tests/test_init_memory_spec.py @@ -0,0 +1,41 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The --memory specification accepted by 'kerf init'.""" + +import pytest + +from kerf.init.main import parse_memory_request + +GB = 1 << 30 + + +def test_plain_size_is_any_node(): + assert parse_memory_request("2GB") == {-1: 2 * GB} + + +def test_per_node_sizes(): + assert parse_memory_request("node0:8GB, node1:512MB") == {0: 8 * GB, 1: 512 << 20} + + +@pytest.mark.parametrize("spec", ["node0:1GB,2GB", "node0:1GB,node0:1GB", "nodeX:1GB", ""]) +def test_invalid_specs(spec): + with pytest.raises(ValueError): + parse_memory_request(spec) + + +@pytest.mark.parametrize("spec", ["0", "node0:0", "4097", "node1:5000"]) +def test_sizes_must_be_positive_and_page_aligned(spec): + with pytest.raises(ValueError): + parse_memory_request(spec) diff --git a/tests/test_init_reconcile.py b/tests/test_init_reconcile.py new file mode 100644 index 0000000..808bec0 --- /dev/null +++ b/tests/test_init_reconcile.py @@ -0,0 +1,152 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The decision 'kerf init' makes between writing a baseline and applying a diff.""" + +import contextlib + +import libfdt + +from kerf.dtc.overlay import OverlayGenerator +from kerf.init import main +from kerf.init.main import reconcile_pool +from kerf.models import ( + CPUAllocation, + GlobalDeviceTree, + HardwareInventory, + MemoryAllocation, + PoolMemoryRegion, +) + +GB = 1 << 30 + + +class FakeManager: + """Stands in for DeviceTreeManager: records the blob, never touches sysfs.""" + + def __init__(self): + self.overlay_gen = OverlayGenerator() + self.applied = [] + + @contextlib.contextmanager + def _acquire_lock(self): + yield + + def apply_dtbo(self, dtbo_data): + self.applied.append(bytes(dtbo_data)) + return "1" + + +class FakeBaselineManager: + def __init__(self, live=None): + self.written = [] + self.live = live + + def write_baseline(self, tree): + self.written.append(tree) + + def read_baseline(self): + return self.live + + +def _tree(cpus, regions, requested): + return GlobalDeviceTree( + hardware=HardwareInventory( + cpus=CPUAllocation(total=16, host_reserved=[0], available=cpus), + memory=MemoryAllocation( + total_bytes=8 * GB, + host_reserved_bytes=0, + regions=regions, + requested=requested, + ), + devices={}, + ), + instances={}, + device_references={}, + ) + + +def test_first_init_writes_the_baseline(): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + requested = _tree([4, 5], [], {-1: 2 * GB}) + + assert reconcile_pool(None, requested, set(), False, manager, baseline_mgr) is None + assert baseline_mgr.written == [requested] + assert not manager.applied + + +def test_first_init_dry_run_writes_nothing(): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + empty = _tree([], [], {}) + + assert reconcile_pool(empty, _tree([4], [], {-1: GB}), set(), True, manager, baseline_mgr) is None + assert not baseline_mgr.written + assert not manager.applied + + +def test_matching_pool_is_a_no_op(): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + current = _tree([4, 5], [PoolMemoryRegion(0x1_0000_0000, 2 * GB, 0)], {}) + requested = _tree([4, 5], [], {0: 2 * GB}) + + diff = reconcile_pool(current, requested, set(), False, manager, baseline_mgr) + + assert diff.is_empty() + assert not manager.applied + assert not baseline_mgr.written + + +def test_dry_run_reports_without_applying(): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + current = _tree([4, 5], [], {}) + requested = _tree([4, 5, 6], [], {0: GB}) + + diff = reconcile_pool(current, requested, set(), True, manager, baseline_mgr) + + assert diff.cpus_to_pool == [6] + assert diff.memory_to_pool == [(0, GB)] + assert not manager.applied + + +def test_apply_writes_a_pool_overlay(): + manager = FakeManager() + current = _tree([4, 5], [PoolMemoryRegion(0x1_0000_0000, GB, 1)], {}) + requested = _tree([4], [], {0: GB}) + baseline_mgr = FakeBaselineManager(live=_tree([4], [PoolMemoryRegion(0x2_0000_0000, GB, 0)], {})) + + diff = reconcile_pool(current, requested, set(), False, manager, baseline_mgr) + + assert diff.cpus_to_host == [5] + assert len(manager.applied) == 1 + fdt = libfdt.Fdt(manager.applied[0]) + overlay = fdt.path_offset("/fragment@0/__overlay__") + assert fdt.getprop(fdt.path_offset("/fragment@0"), "target-path").as_str() == "/resources" + assert fdt.subnode_offset(overlay, "cpu-remove") >= 0 + assert fdt.subnode_offset(overlay, "memory-add") >= 0 + assert fdt.subnode_offset(overlay, "memory-remove") >= 0 + assert not baseline_mgr.written + + +def test_teardown_returns_everything(monkeypatch): + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1, 4, 5}) + manager, baseline_mgr = FakeManager(), FakeBaselineManager(live=_tree([], [], {})) + current = _tree([4, 5], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + + diff = main.reconcile_pool(current, main.build_teardown_tree(), set(), False, + manager, baseline_mgr) + + assert diff.cpus_to_host == [4, 5] + assert [r.base for r in diff.memory_to_host] == [0x1_0000_0000] + assert diff.memory_to_pool == [] + assert len(manager.applied) == 1 From 3da9f29370915426acf6301d894cf8e94c34965d Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 12:12:47 -0700 Subject: [PATCH 10/23] init: Detect a live pool from the kernel read-back, not the host CPU list Before a pool exists the kernel still publishes a root device tree, and it describes the host: the online CPUs land in cpus with no cpus- available and no memory node, so reading it back raised a ParseError that aborted the very first init, and the CPU list alone made an empty pool look live. Treat an unreadable read-back as no pool yet and take liveness from cpus-available, the pool chunks and the pool devices, which only the pool branch of the kernel's tree emits; a teardown of an already empty pool no longer writes a baseline the kernel would reject, one that would strand instances names them instead of failing the transaction, and a read-back that fails after the transaction landed is a note rather than an error. Signed-off-by: Cong Wang --- src/kerf/init/main.py | 94 +++++++++++++++++++++++++++++++----- src/kerf/resources.py | 54 +++++++++++---------- tests/test_init_reconcile.py | 64 ++++++++++++++++++++++-- 3 files changed, 172 insertions(+), 40 deletions(-) diff --git a/src/kerf/init/main.py b/src/kerf/init/main.py index 4f25979..e04fb2a 100644 --- a/src/kerf/init/main.py +++ b/src/kerf/init/main.py @@ -25,7 +25,7 @@ import re import sys from pathlib import Path -from typing import Dict, Optional +from typing import Dict, List, Optional import click import libfdt @@ -38,7 +38,6 @@ from ..baseline import BaselineManager from ..create.main import parse_cpu_spec, parse_device_list, parse_memory_spec from ..dtc.parser import DeviceTreeParser -from ..resources import get_busy_chunks_from_iomem from ..dtc.reporter import ValidationReporter from ..dtc.validator import MultikernelValidator from ..exceptions import KernelInterfaceError, ParseError, ValidationError @@ -50,6 +49,7 @@ MemoryAllocation, ) from ..pool_diff import PoolDiff, compute_pool_diff +from ..resources import get_busy_chunks_from_iomem from ..runtime import DeviceTreeManager @@ -545,12 +545,60 @@ def build_teardown_tree() -> GlobalDeviceTree: return GlobalDeviceTree(hardware=hardware, instances={}, device_references={}) +INSTANCES_DIR = "/sys/fs/multikernel/instances" + + +def list_instance_names(instances_dir: str = INSTANCES_DIR) -> List[str]: + """Names of the instances the kernel currently holds.""" + path = Path(instances_dir) + if not path.exists(): + return [] + try: + return sorted(child.name for child in path.iterdir() if child.is_dir()) + except OSError: + return [] + + def pool_is_live(current: Optional[GlobalDeviceTree]) -> bool: - """Whether the kernel already holds pool resources we have to diff against.""" + """ + Whether the kernel already holds pool resources we have to diff against. + + A non-empty cpus list is not evidence: before a pool exists the root + read-back lists the host's own online CPUs there. Only the pool branch + of the kernel's device tree emits cpus-available, memory@N or devices. + """ if current is None: return False hardware = current.hardware - return bool(hardware.cpus.available or hardware.memory.regions or hardware.devices) + return bool(hardware.cpus.available_free is not None + or hardware.memory.regions + or hardware.devices) + + +def read_current_pool(baseline_mgr) -> Optional[GlobalDeviceTree]: + """ + Read the live pool back from the kernel. + + Before any pool exists the kernel still publishes a root device tree, + but one that describes the host rather than a pool and that carries no + memory node, so a failed read means "no pool yet", not an error. + + Args: + baseline_mgr: BaselineManager to read through + + Returns: + The baseline the kernel reports, or None if there is no pool yet + """ + try: + return baseline_mgr.read_baseline() + except (ParseError, KernelInterfaceError): + return None + + +def request_is_empty(requested: GlobalDeviceTree) -> bool: + """Whether the request asks for nothing at all, as --teardown does.""" + hardware = requested.hardware + return not (hardware.cpus.available or hardware.memory.requested or hardware.devices) def _print_diff(diff: PoolDiff) -> None: @@ -614,8 +662,14 @@ def reconcile_pool( Raises: KernelInterfaceError: If the kernel rejects the write or the overlay + ValidationError: If a teardown would strand running instances """ if not pool_is_live(current): + # The kernel rejects a baseline with no memory@N node, and there is + # nothing to hand back anyway. + if request_is_empty(requested): + click.echo("Pool is already empty; nothing to do") + return None if dry_run: click.echo("Baseline validation passed; would write the initial baseline (dry-run)") return None @@ -632,10 +686,26 @@ def reconcile_pool( click.echo("Would apply the changes above (dry-run)") return diff + if request_is_empty(requested): + held = list_instance_names() + if held: + raise ValidationError( + f"The pool still runs {len(held)} instance(s): " + f"delete instances {', '.join(held)} first" + ) + with manager._acquire_lock(): # pylint: disable=protected-access tx_id = manager.apply_dtbo(manager.overlay_gen.generate_pool_overlay(diff)) click.echo(f"✓ Pool updated (transaction {tx_id})") - _report_shortfall(baseline_mgr.read_baseline(), requested) + + try: + live = baseline_mgr.read_baseline() + except (ParseError, KernelInterfaceError) as e: + # The transaction already landed; a failed read-back is worth a word, + # not a failure. + click.echo(f"Note: could not read the pool back after the transaction: {e}", err=True) + else: + _report_shortfall(live, requested) return diff @@ -663,8 +733,8 @@ def _dump_baseline_dts(baseline_mgr: BaselineManager, tree: GlobalDeviceTree) -> @click.option('--memory', '-m', help='Pool memory: SIZE for any node (e.g. "2GB") or per-node "node0:8GB,node1:8GB". Required with --cpus, mutually exclusive with --input.') @click.option('--devices', '-d', help='Device names (comma-separated, e.g., "enp9s0_dev,nvme0"). Mutually exclusive with --input. Creates minimal device entries in baseline.') @click.option('--teardown', is_flag=True, help='Return every pool resource to the host. Mutually exclusive with --input, --cpus, --memory and --devices.') -@click.option('--dry-run', is_flag=True, help='Validate without applying') -@click.option('--report', is_flag=True, help='Generate detailed validation report') +@click.option('--dry-run', is_flag=True, help='Report the plan without applying it. Still reads the pool from the kernel, so it needs root.') +@click.option('--report', is_flag=True, help='Generate detailed validation report. Ignored with --teardown.') @click.option('--format', type=click.Choice(['text', 'json', 'yaml']), default='text', help='Report format (default: text)') @click.option('--verbose', '-v', is_flag=True, help='Verbose output') @@ -680,7 +750,8 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: pool the kernel reports, and only the difference is applied. An empty pool takes the baseline write; a live pool is reconciled with a /resources overlay transaction. Use --teardown to return everything to - the host, and --dry-run to see the plan without applying it. + the host, and --dry-run to see the plan without applying it. Even + --dry-run reads the pool from the kernel, so every form needs root. You can either provide a DTS/DTB file via --input, or construct the request from command line arguments using --cpus and --memory. These @@ -821,9 +892,7 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: manager = DeviceTreeManager() mount_multikernel_fs(verbose=verbose) - current = None - if baseline_mgr.baseline_path.exists() and baseline_mgr.baseline_path.stat().st_size: - current = baseline_mgr.read_baseline() + current = read_current_pool(baseline_mgr) if debug and not pool_is_live(current): _dump_baseline_dts(baseline_mgr, tree) @@ -831,6 +900,9 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: try: reconcile_pool(current, tree, get_busy_chunks_from_iomem(), dry_run, manager, baseline_mgr) + except ValidationError as e: + click.echo(f"Error: {e}", err=True) + sys.exit(1) except KernelInterfaceError as e: click.echo(f"Error: {e}", err=True) click.echo("Run 'dmesg | tail' for the kernel's reason (it names the busy CPU or chunk).", err=True) diff --git a/src/kerf/resources.py b/src/kerf/resources.py index bb19d81..06050b9 100644 --- a/src/kerf/resources.py +++ b/src/kerf/resources.py @@ -54,6 +54,29 @@ def _parse_iomem_regions(iomem_path: str) -> List[Tuple[int, int, str]]: return regions +def _pool_chunks(regions: List[Tuple[int, int, str]]) -> List[Tuple[int, int]]: + """Pick the pool chunks out of already parsed /proc/iomem regions.""" + return [ + (base, end - base + 1) + for base, end, name in regions + if MULTIKERNEL_POOL_NAME in name + ] + + +def _chunk_children(regions: List[Tuple[int, int, str]], + chunks: List[Tuple[int, int]]) -> List[Tuple[int, int, int]]: + """Map every region nested in a pool chunk to (chunk_base, base, end).""" + children = [] + for base, end, name in regions: + if MULTIKERNEL_POOL_NAME in name: + continue + for chunk_base, chunk_size in chunks: + if chunk_base <= base and end <= chunk_base + chunk_size - 1: + children.append((chunk_base, base, end)) + break + return children + + def get_pool_chunks_from_iomem(iomem_path: str = IOMEM_PATH) -> List[Tuple[int, int]]: """ List every multikernel pool chunk registered in /proc/iomem. @@ -61,11 +84,7 @@ def get_pool_chunks_from_iomem(iomem_path: str = IOMEM_PATH) -> List[Tuple[int, Returns: (base_address, size_bytes) tuples in /proc/iomem order """ - return [ - (base, end - base + 1) - for base, end, name in _parse_iomem_regions(iomem_path) - if MULTIKERNEL_POOL_NAME in name - ] + return _pool_chunks(_parse_iomem_regions(iomem_path)) def get_busy_chunks_from_iomem(iomem_path: str = IOMEM_PATH) -> Set[int]: @@ -78,15 +97,8 @@ def get_busy_chunks_from_iomem(iomem_path: str = IOMEM_PATH) -> Set[int]: Returns: Base addresses of the chunks with at least one child region """ - chunks = get_pool_chunks_from_iomem(iomem_path) - busy = set() - for base, end, name in _parse_iomem_regions(iomem_path): - if MULTIKERNEL_POOL_NAME in name: - continue - for chunk_base, chunk_size in chunks: - if chunk_base <= base and end <= chunk_base + chunk_size - 1: - busy.add(chunk_base) - return busy + regions = _parse_iomem_regions(iomem_path) + return {chunk_base for chunk_base, _, _ in _chunk_children(regions, _pool_chunks(regions))} def get_memory_pool_from_iomem(iomem_path: str = IOMEM_PATH) -> Optional[Tuple[int, int]]: @@ -112,22 +124,14 @@ def get_pool_allocated_bytes(iomem_path: str = IOMEM_PATH) -> Optional[Tuple[int (first_chunk_base, pool_bytes, allocated_bytes), or None if the pool is not registered in /proc/iomem """ - chunks = get_pool_chunks_from_iomem(iomem_path) + regions = _parse_iomem_regions(iomem_path) + chunks = _pool_chunks(regions) if not chunks: return None - children = [] - for base, end, name in _parse_iomem_regions(iomem_path): - if MULTIKERNEL_POOL_NAME in name: - continue - for chunk_base, chunk_size in chunks: - if base >= chunk_base and end <= chunk_base + chunk_size - 1: - children.append((base, end)) - break - allocated = 0 current_base = current_end = None - for base, end in sorted(children): + for _, base, end in sorted(_chunk_children(regions, chunks), key=lambda c: c[1:]): if current_base is None: current_base, current_end = base, end elif base <= current_end + 1: diff --git a/tests/test_init_reconcile.py b/tests/test_init_reconcile.py index 808bec0..5eb8de1 100644 --- a/tests/test_init_reconcile.py +++ b/tests/test_init_reconcile.py @@ -17,8 +17,10 @@ import contextlib import libfdt +import pytest from kerf.dtc.overlay import OverlayGenerator +from kerf.exceptions import ParseError, ValidationError from kerf.init import main from kerf.init.main import reconcile_pool from kerf.models import ( @@ -49,21 +51,27 @@ def apply_dtbo(self, dtbo_data): class FakeBaselineManager: - def __init__(self, live=None): + """Stands in for BaselineManager: records writes, replays one read-back.""" + + def __init__(self, live=None, read_error=None): self.written = [] self.live = live + self.read_error = read_error def write_baseline(self, tree): self.written.append(tree) def read_baseline(self): + if self.read_error is not None: + raise self.read_error return self.live -def _tree(cpus, regions, requested): +def _tree(cpus, regions, requested, available_free=None): return GlobalDeviceTree( hardware=HardwareInventory( - cpus=CPUAllocation(total=16, host_reserved=[0], available=cpus), + cpus=CPUAllocation(total=16, host_reserved=[0], available=cpus, + available_free=available_free), memory=MemoryAllocation( total_bytes=8 * GB, host_reserved_bytes=0, @@ -109,7 +117,7 @@ def test_matching_pool_is_a_no_op(): def test_dry_run_reports_without_applying(): manager, baseline_mgr = FakeManager(), FakeBaselineManager() - current = _tree([4, 5], [], {}) + current = _tree([4, 5], [], {}, available_free=[4, 5]) requested = _tree([4, 5, 6], [], {0: GB}) diff = reconcile_pool(current, requested, set(), True, manager, baseline_mgr) @@ -140,6 +148,7 @@ def test_apply_writes_a_pool_overlay(): def test_teardown_returns_everything(monkeypatch): monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1, 4, 5}) + monkeypatch.setattr(main, "list_instance_names", lambda: []) manager, baseline_mgr = FakeManager(), FakeBaselineManager(live=_tree([], [], {})) current = _tree([4, 5], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) @@ -150,3 +159,50 @@ def test_teardown_returns_everything(monkeypatch): assert [r.base for r in diff.memory_to_host] == [0x1_0000_0000] assert diff.memory_to_pool == [] assert len(manager.applied) == 1 + + +def test_unparseable_read_back_is_a_first_init(): + manager = FakeManager() + baseline_mgr = FakeBaselineManager( + read_error=ParseError("No memory description in /resources")) + requested = _tree([4, 5], [], {-1: GB}) + + current = main.read_current_pool(baseline_mgr) + + assert current is None + assert reconcile_pool(current, requested, set(), False, manager, baseline_mgr) is None + assert baseline_mgr.written == [requested] + assert not manager.applied + + +def test_host_cpu_list_alone_is_not_a_live_pool(): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + # What the kernel publishes before a pool exists: its own online CPUs, + # no cpus-available, no memory@N. + current = _tree([0, 1, 2, 3], [], {}) + requested = _tree([4, 5], [], {-1: GB}) + + assert not main.pool_is_live(current) + assert reconcile_pool(current, requested, set(), False, manager, baseline_mgr) is None + assert baseline_mgr.written == [requested] + assert not manager.applied + + +def test_teardown_of_an_empty_pool_writes_nothing(): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + + assert reconcile_pool(_tree([0, 1], [], {}), main.build_teardown_tree(), set(), False, + manager, baseline_mgr) is None + assert not baseline_mgr.written + assert not manager.applied + + +def test_teardown_refuses_while_instances_exist(monkeypatch): + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 4}) + monkeypatch.setattr(main, "list_instance_names", lambda: ["db", "web"]) + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + current = _tree([4], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + + with pytest.raises(ValidationError, match="delete instances db, web first"): + reconcile_pool(current, main.build_teardown_tree(), set(), False, manager, baseline_mgr) + assert not manager.applied From 3d938a074a0a93d9e481902caabad721f81b8b7f Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 12:24:34 -0700 Subject: [PATCH 11/23] show: List every pool chunk with its NUMA node kerf show printed a single pool base and size, which cannot describe a pool made of several chunks on different nodes. List every chunk with its base, size and node, show pool membership and the free subset of CPUs separately now that the kernel reports both, hide the total and host-reserved lines when the read-back carries no such numbers, and stop the validation reporter from dividing by a zero total. Signed-off-by: Cong Wang --- src/kerf/dtc/reporter.py | 13 ++-- src/kerf/show/main.py | 47 +++++++------ tests/test_kerf.py | 15 +++++ tests/test_show.py | 138 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 189 insertions(+), 24 deletions(-) create mode 100644 tests/test_show.py diff --git a/src/kerf/dtc/reporter.py b/src/kerf/dtc/reporter.py index 3becb16..54ee2f3 100644 --- a/src/kerf/dtc/reporter.py +++ b/src/kerf/dtc/reporter.py @@ -123,13 +123,18 @@ def _format_hardware_inventory(self, tree: GlobalDeviceTree) -> List[str]: total_gb = memory.total_bytes / (1024**3) host_gb = memory.host_reserved_bytes / (1024**3) memory_pool_gb = memory.memory_pool_bytes / (1024**3) - host_mem_percent = (memory.host_reserved_bytes / memory.total_bytes) * 100 - memory_pool_mem_percent = (memory.memory_pool_bytes / memory.total_bytes) * 100 + # total_bytes is 0 when the tree only carries pool chunks read back + # from the kernel, not a system-wide total. + if memory.total_bytes: + host_mem_percent = f"{(memory.host_reserved_bytes / memory.total_bytes) * 100:.0f}%" + memory_pool_mem_percent = f"{(memory.memory_pool_bytes / memory.total_bytes) * 100:.0f}%" + else: + host_mem_percent = memory_pool_mem_percent = "n/a" lines.append(f" Memory: {total_gb:.0f}GB total") - lines.append(f" Host reserved: {host_gb:.0f}GB ({host_mem_percent:.0f}%)") + lines.append(f" Host reserved: {host_gb:.0f}GB ({host_mem_percent})") lines.append( - f" Memory pool: {memory_pool_gb:.0f}GB at {hex(memory.memory_pool_base)} ({memory_pool_mem_percent:.0f}%)" + f" Memory pool: {memory_pool_gb:.0f}GB at {hex(memory.memory_pool_base)} ({memory_pool_mem_percent})" ) # Device information diff --git a/src/kerf/show/main.py b/src/kerf/show/main.py index 0fa515f..5119517 100644 --- a/src/kerf/show/main.py +++ b/src/kerf/show/main.py @@ -281,9 +281,17 @@ def display_baseline_info(tree: GlobalDeviceTree, verbose: bool = False): click.echo( f" Host Reserved: {len(hardware.cpus.host_reserved)} cpus: {hardware.cpus.host_reserved}" ) - click.echo( - f" Available: {len(hardware.cpus.available)} cpus: {hardware.cpus.available}" - ) + if hardware.cpus.available_free is not None: + click.echo( + f" Pool CPUs: {len(hardware.cpus.available)} cpus: {hardware.cpus.available}" + ) + click.echo( + f" Available CPUs: {len(hardware.cpus.available_free)} cpus: {hardware.cpus.available_free}" + ) + else: + click.echo( + f" Available: {len(hardware.cpus.available)} cpus: {hardware.cpus.available}" + ) if verbose and hardware.cpus.topology: click.echo("\n Topology:") @@ -294,29 +302,28 @@ def display_baseline_info(tree: GlobalDeviceTree, verbose: bool = False): # Memory Information click.echo("\n Memory:") - total_gb = hardware.memory.total_bytes / (1024**3) - reserved_gb = hardware.memory.host_reserved_bytes / (1024**3) - click.echo(f" Total: {total_gb:.2f} GB ({hardware.memory.total_bytes} bytes)") - click.echo( - f" Host Reserved: {reserved_gb:.2f} GB ({hardware.memory.host_reserved_bytes} bytes)" - ) + # total_bytes/host_reserved_bytes are 0 when the tree only carries pool + # sizes read back from the kernel, not a system-wide total. + if hardware.memory.total_bytes: + total_gb = hardware.memory.total_bytes / (1024**3) + click.echo(f" Total: {total_gb:.2f} GB ({hardware.memory.total_bytes} bytes)") + if hardware.memory.host_reserved_bytes: + reserved_gb = hardware.memory.host_reserved_bytes / (1024**3) + click.echo( + f" Host Reserved: {reserved_gb:.2f} GB ({hardware.memory.host_reserved_bytes} bytes)" + ) # /proc/iomem is the source of truth for the pool chunks and the # instance allocations carved out of them; the baseline tree only # snapshots the pool as of the last transaction. + click.echo("\n Memory Pool:") + for region in hardware.memory.regions: + node = f" node {region.node}" if region.node >= 0 else "" + click.echo(f" Chunk: {hex(region.base)} {region.size / (1024**3):.2f} GB{node}") + usage = get_pool_allocated_bytes() if usage is not None: - pool_base, pool_bytes, allocated_bytes = usage - else: - pool_base = hardware.memory.memory_pool_base - pool_bytes = hardware.memory.memory_pool_bytes - allocated_bytes = None - - pool_gb = pool_bytes / (1024**3) - click.echo(f" Pool Base: 0x{pool_base:x}") - click.echo(f" Pool Size: {pool_gb:.2f} GB ({pool_bytes} bytes)") - click.echo(f" Pool End: 0x{pool_base + pool_bytes:x}") - if allocated_bytes is not None: + _, pool_bytes, allocated_bytes = usage available_bytes = pool_bytes - allocated_bytes allocated_gb = allocated_bytes / (1024**3) available_gb = available_bytes / (1024**3) diff --git a/tests/test_kerf.py b/tests/test_kerf.py index a1917e1..c38ed31 100644 --- a/tests/test_kerf.py +++ b/tests/test_kerf.py @@ -133,6 +133,21 @@ def test_validation(): assert result.is_valid, "Validation should pass for valid tree" +def test_reporter_zero_total_bytes(): + """A kernel read-back tree has total_bytes == 0; percentages must not divide by zero.""" + tree = create_test_tree() + tree.hardware.memory.total_bytes = 0 + tree.hardware.memory.host_reserved_bytes = 0 + + validator = MultikernelValidator() + result = validator.validate(tree) + + reporter = ValidationReporter() + report = reporter.generate_report(result, tree, verbose=True) + + assert "n/a" in report + + def test_extraction(): """Test instance extraction functionality.""" print("\n=== Testing Instance Extraction ===") diff --git a/tests/test_show.py b/tests/test_show.py new file mode 100644 index 0000000..8e31499 --- /dev/null +++ b/tests/test_show.py @@ -0,0 +1,138 @@ +# Copyright 2026 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Tests for `kerf show` baseline hardware display. +""" + +from kerf.models import ( + CPUAllocation, + GlobalDeviceTree, + HardwareInventory, + MemoryAllocation, + PoolMemoryRegion, +) +from kerf.show.main import display_baseline_info + + +def _tree(cpus, memory): + hardware = HardwareInventory(cpus=cpus, memory=memory, devices={}) + return GlobalDeviceTree(hardware=hardware, instances={}, device_references={}) + + +def _cpus(available_free=None): + return CPUAllocation( + total=32, + host_reserved=[0, 1, 2, 3], + available=list(range(4, 32)), + available_free=available_free, + ) + + +def test_lists_every_pool_chunk_with_its_node(capsys, monkeypatch): + memory = MemoryAllocation( + total_bytes=16 * 1024**3, + host_reserved_bytes=2 * 1024**3, + regions=[ + PoolMemoryRegion(base=0x100000000, size=4 * 1024**3, node=0), + PoolMemoryRegion(base=0x200000000, size=6 * 1024**3, node=1), + PoolMemoryRegion(base=0x300000000, size=1 * 1024**3, node=-1), + ], + ) + monkeypatch.setattr( + "kerf.show.main.get_pool_allocated_bytes", + lambda: (0x100000000, 11 * 1024**3, 3 * 1024**3), + ) + + display_baseline_info(_tree(_cpus(), memory)) + out = capsys.readouterr().out + + assert "Chunk: 0x100000000 4.00 GB node 0" in out + assert "Chunk: 0x200000000 6.00 GB node 1" in out + assert "Chunk: 0x300000000 1.00 GB" in out + assert "0x300000000 1.00 GB node" not in out + assert "Pool Allocated: 3.00 GB" in out + assert "Pool Available: 8.00 GB" in out + + +def test_no_allocation_summary_without_iomem(capsys, monkeypatch): + memory = MemoryAllocation( + total_bytes=0, + host_reserved_bytes=0, + regions=[PoolMemoryRegion(base=0x100000000, size=4 * 1024**3, node=0)], + ) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(_cpus(), memory)) + out = capsys.readouterr().out + + assert "Pool Allocated" not in out + assert "Pool Available" not in out + + +def test_memory_total_and_host_reserved_hidden_when_zero(capsys, monkeypatch): + memory = MemoryAllocation( + total_bytes=0, + host_reserved_bytes=0, + regions=[PoolMemoryRegion(base=0x100000000, size=4 * 1024**3, node=0)], + ) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(_cpus(), memory)) + out = capsys.readouterr().out + + # The CPU section always prints its own Total/Host Reserved; only the + # memory ones (GB-suffixed) are suppressed when the value is 0. + assert "GB) " not in out + for line in out.splitlines(): + assert not line.strip().startswith("Total:") or "GB" not in line + assert not line.strip().startswith("Host Reserved:") or "GB" not in line + + +def test_memory_total_and_host_reserved_shown_when_nonzero(capsys, monkeypatch): + memory = MemoryAllocation( + total_bytes=16 * 1024**3, + host_reserved_bytes=2 * 1024**3, + regions=[PoolMemoryRegion(base=0x100000000, size=4 * 1024**3, node=0)], + ) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(_cpus(), memory)) + out = capsys.readouterr().out + + assert "Total: 16.00 GB" in out + assert "Host Reserved: 2.00 GB" in out + + +def test_cpu_pool_and_available_split_when_free_subset_known(capsys, monkeypatch): + memory = MemoryAllocation(total_bytes=0, host_reserved_bytes=0, regions=[]) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(_cpus(available_free=list(range(4, 20))), memory)) + out = capsys.readouterr().out + + assert "Pool CPUs: 28 cpus" in out + assert "Available CPUs: 16 cpus" in out + + +def test_cpu_available_line_kept_when_free_subset_unknown(capsys, monkeypatch): + memory = MemoryAllocation(total_bytes=0, host_reserved_bytes=0, regions=[]) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(_cpus(available_free=None), memory)) + out = capsys.readouterr().out + + assert "Available: 28 cpus" in out + assert "Pool CPUs" not in out + assert "Available CPUs" not in out From abf06cb5c147dabfa6cd954340702b31030ed9a6 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 12:37:25 -0700 Subject: [PATCH 12/23] init: Keep pool CPUs valid when re-initializing a live pool The host drops a CPU from /proc/cpuinfo once the pool owns it, so every re-init of a live pool rejected the very CPUs it was asked to keep. Read the pool before building the requested tree and treat its CPUs as valid APIC ids, and skip the post-teardown read-back that always failed because an emptied pool has no memory@N left to report. Signed-off-by: Cong Wang --- src/kerf/init/main.py | 42 ++++++++++++++++++++-------- tests/test_init_reconcile.py | 53 ++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 12 deletions(-) diff --git a/src/kerf/init/main.py b/src/kerf/init/main.py index e04fb2a..a1c2afb 100644 --- a/src/kerf/init/main.py +++ b/src/kerf/init/main.py @@ -408,7 +408,8 @@ def build_baseline_from_cmdline( cpus: str, memory: Optional[str] = None, devices: Optional[str] = None, - verbose: bool = False + verbose: bool = False, + pool_cpus: Optional[set] = None ) -> GlobalDeviceTree: """ Build a GlobalDeviceTree from command line arguments. @@ -419,6 +420,7 @@ def build_baseline_from_cmdline( "node0:8GB,node1:8GB" for specific nodes devices: Optional device names (comma-separated, e.g., "enp9s0_dev,nvme0") verbose: Whether to print verbose output + pool_cpus: APIC IDs the pool already holds Returns: GlobalDeviceTree with resources only (no instances) @@ -444,6 +446,10 @@ def build_baseline_from_cmdline( "Ensure the system exposes CPU topology information." ) + # The host stops listing a CPU in /proc/cpuinfo once the pool takes it, + # so a re-init has to count the pool's own CPUs as valid. + valid_apic_ids = valid_apic_ids | set(pool_cpus or ()) + invalid_cpus = set(cpu_list) - valid_apic_ids if invalid_cpus: raise ValueError( @@ -527,9 +533,9 @@ def build_baseline_from_cmdline( return tree -def build_teardown_tree() -> GlobalDeviceTree: +def build_teardown_tree(pool_cpus: Optional[set] = None) -> GlobalDeviceTree: """Build the empty requested state: every pool resource goes back to the host.""" - apic_ids = get_valid_apic_ids_from_system() or set() + apic_ids = (get_valid_apic_ids_from_system() or set()) | set(pool_cpus or ()) cpu_allocation = CPUAllocation( total=(max(apic_ids) + 1) if apic_ids else 0, host_reserved=sorted(apic_ids), @@ -575,6 +581,13 @@ def pool_is_live(current: Optional[GlobalDeviceTree]) -> bool: or hardware.devices) +def pool_apic_ids(current: Optional[GlobalDeviceTree]) -> set: + """APIC IDs the live pool holds, which the host no longer reports.""" + if not pool_is_live(current): + return set() + return set(current.hardware.cpus.available or []) + + def read_current_pool(baseline_mgr) -> Optional[GlobalDeviceTree]: """ Read the live pool back from the kernel. @@ -698,6 +711,11 @@ def reconcile_pool( tx_id = manager.apply_dtbo(manager.overlay_gen.generate_pool_overlay(diff)) click.echo(f"✓ Pool updated (transaction {tx_id})") + if request_is_empty(requested): + # /resources carries no memory@N once the pool is gone, so there is + # nothing left to read back or compare against. + return diff + try: live = baseline_mgr.read_baseline() except (ParseError, KernelInterfaceError) as e: @@ -809,8 +827,14 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: parser = DeviceTreeParser() dts_content = None + baseline_mgr = BaselineManager() + manager = DeviceTreeManager() + mount_multikernel_fs(verbose=verbose) + current = read_current_pool(baseline_mgr) + live_cpus = pool_apic_ids(current) + if teardown: - tree = build_teardown_tree() + tree = build_teardown_tree(pool_cpus=live_cpus) elif input: # Parse from input file input_path = Path(input) @@ -832,7 +856,8 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: else: # Build from command line arguments try: - tree = build_baseline_from_cmdline(cpus, memory=memory, devices=devices, verbose=verbose) + tree = build_baseline_from_cmdline(cpus, memory=memory, devices=devices, + verbose=verbose, pool_cpus=live_cpus) except ValueError as e: click.echo(f"Error: {e}", err=True) sys.exit(2) @@ -840,8 +865,6 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: click.echo(f"Error: {e}", err=True) sys.exit(1) - baseline_mgr = BaselineManager() - try: baseline_mgr.validate_baseline(tree) except ValidationError as e: @@ -889,11 +912,6 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: debug = ctx.obj.get('debug', False) if ctx and ctx.obj else False - manager = DeviceTreeManager() - mount_multikernel_fs(verbose=verbose) - - current = read_current_pool(baseline_mgr) - if debug and not pool_is_live(current): _dump_baseline_dts(baseline_mgr, tree) diff --git a/tests/test_init_reconcile.py b/tests/test_init_reconcile.py index 5eb8de1..aaee41f 100644 --- a/tests/test_init_reconcile.py +++ b/tests/test_init_reconcile.py @@ -206,3 +206,56 @@ def test_teardown_refuses_while_instances_exist(monkeypatch): with pytest.raises(ValidationError, match="delete instances db, web first"): reconcile_pool(current, main.build_teardown_tree(), set(), False, manager, baseline_mgr) assert not manager.applied + + +def test_pool_cpus_stay_valid_apic_ids(monkeypatch): + # A CPU in the pool is gone from /proc/cpuinfo, but re-init must still name it. + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0}) + live = _tree([1, 2, 3], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + + tree = main.build_baseline_from_cmdline( + "1-3", memory="512MB", pool_cpus=main.pool_apic_ids(live)) + + assert tree.hardware.cpus.available == [1, 2, 3] + assert tree.hardware.cpus.host_reserved == [0] + assert tree.hardware.cpus.total == 4 + + +def test_apic_id_outside_system_and_pool_is_rejected(monkeypatch): + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0}) + live = _tree([1], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + + with pytest.raises(ValueError, match=r"Invalid APIC ID\(s\) specified: \[9\]"): + main.build_baseline_from_cmdline("1,9", memory="512MB", + pool_cpus=main.pool_apic_ids(live)) + + +def test_pool_apic_ids_ignores_a_host_only_read_back(): + assert main.pool_apic_ids(None) == set() + assert main.pool_apic_ids(_tree([0, 1, 2, 3], [], {})) == set() + + +def test_teardown_tree_reserves_pool_cpus(monkeypatch): + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0}) + + tree = main.build_teardown_tree(pool_cpus={1, 2, 3}) + + assert tree.hardware.cpus.host_reserved == [0, 1, 2, 3] + assert tree.hardware.cpus.available == [] + assert tree.hardware.cpus.total == 4 + + +def test_teardown_does_not_read_the_pool_back(monkeypatch, capsys): + # After a teardown /resources has no memory@N, so a read-back always + # fails; complaining about it makes a clean teardown look broken. + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1}) + monkeypatch.setattr(main, "list_instance_names", lambda: []) + manager = FakeManager() + baseline_mgr = FakeBaselineManager( + read_error=ParseError("No memory description in /resources")) + current = _tree([1], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + + main.reconcile_pool(current, main.build_teardown_tree(pool_cpus={1}), set(), False, + manager, baseline_mgr) + + assert "could not read the pool back" not in capsys.readouterr().err From 301ded3f85df5d6606212b68a7963afbc2a6be34 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 13:14:20 -0700 Subject: [PATCH 13/23] pool: Validate and allocate against every pool chunk The pool is a list of chunks, but validation still compared the read- back against the first chunk in /proc/iomem and bounded instance memory by base plus sum, so a two-chunk pool failed validation and kerf create aborted before reaching the kernel. Compare the whole chunk set, check that an instance region lies inside one chunk, stop picking a base in kerf create since the kernel places instance memory itself (a base given on the command line is still checked), and let an instance grow only into the chunk it already sits in because the memory-add overlay names an existing range. Signed-off-by: Cong Wang --- src/kerf/__init__.py | 4 +- src/kerf/create/main.py | 26 +++++------ src/kerf/dtc/overlay.py | 1 - src/kerf/dtc/validator.py | 82 ++++++++++++--------------------- src/kerf/resources.py | 94 +++++++++++--------------------------- src/kerf/runtime.py | 9 +--- src/kerf/update/main.py | 38 ++++++--------- tests/test_pool_overlay.py | 5 ++ tests/test_resources.py | 53 ++++++++------------- tests/test_validator.py | 88 +++++++++++++++++++++++++++++++++++ 10 files changed, 199 insertions(+), 201 deletions(-) diff --git a/src/kerf/__init__.py b/src/kerf/__init__.py index 6e21e19..bd3e205 100644 --- a/src/kerf/__init__.py +++ b/src/kerf/__init__.py @@ -41,7 +41,7 @@ get_available_cpus, get_allocated_cpus, get_allocated_memory_regions, - find_available_memory_base, + chunk_containing, validate_cpu_allocation, validate_memory_allocation, find_next_instance_id, @@ -67,7 +67,7 @@ "get_available_cpus", "get_allocated_cpus", "get_allocated_memory_regions", - "find_available_memory_base", + "chunk_containing", "validate_cpu_allocation", "validate_memory_allocation", "find_next_instance_id", diff --git a/src/kerf/create/main.py b/src/kerf/create/main.py index 6ec93be..568949f 100644 --- a/src/kerf/create/main.py +++ b/src/kerf/create/main.py @@ -31,7 +31,6 @@ from ..resources import ( validate_cpu_allocation, validate_memory_allocation, - find_available_memory_base, get_available_cpus, ) from ..exceptions import ValidationError, KernelInterfaceError, ResourceError, ParseError @@ -302,6 +301,12 @@ def parse_memory_spec(memory_spec: str) -> int: ) from exc +def _placement(instance) -> str: + """Name the base only when one was asked for; the kernel picks otherwise.""" + base = instance.resources.memory_base + return f" at {hex(base)}" if base else "" + + def parse_memory_base(base_spec: str) -> int: """ Parse memory base address specification. @@ -432,7 +437,8 @@ def dump_overlay_for_debug( ) @click.option( "--memory-base", - help="Memory base address (hex: 0x80000000 or decimal, auto-assigned if not specified)", + help="Memory base address to request (hex: 0x80000000 or decimal). " + "Only checked against the pool; the kernel places instance memory itself.", ) @click.option( "--devices", @@ -641,17 +647,11 @@ def create_instance_operation(current): # Validate CPU allocation (against baseline and existing instances) validate_cpu_allocation(modified, cpu_list) - # Find memory base if not specified + # The kernel places instance memory itself; a base is only ever + # a caller's request, so it is all there is to validate. if memory_base_addr is None: - found_base = find_available_memory_base(modified, memory_bytes) - if found_base is None: - raise ResourceError( - f"No available memory region found for {memory_bytes} bytes. " - "Try specifying --memory-base or reduce memory size." - ) - memory_base_addr = found_base + memory_base_addr = 0 else: - # Validate specified memory base validate_memory_allocation(modified, memory_base_addr, memory_bytes) # Create instance resources with topology settings @@ -705,7 +705,7 @@ def create_instance_operation(current): click.echo( f" NUMA Nodes: {', '.join(map(str, instance.resources.numa_nodes))}" ) - click.echo(f" Memory: {memory} at {hex(instance.resources.memory_base)}") + click.echo(f" Memory: {memory}{_placement(instance)}") if instance.resources.memory_policy: click.echo(f" Memory Policy: {instance.resources.memory_policy}") if instance.resources.devices: @@ -745,7 +745,7 @@ def create_instance_operation(current): if instance.resources.numa_nodes: numa_str = ", ".join(map(str, instance.resources.numa_nodes)) click.echo(f" NUMA Nodes: {numa_str}") - click.echo(f" Memory: {memory} at {hex(instance.resources.memory_base)}") + click.echo(f" Memory: {memory}{_placement(instance)}") if instance.resources.memory_policy: click.echo(f" Memory Policy: {instance.resources.memory_policy}") if instance.resources.devices: diff --git a/src/kerf/dtc/overlay.py b/src/kerf/dtc/overlay.py index 4135556..1cc1166 100644 --- a/src/kerf/dtc/overlay.py +++ b/src/kerf/dtc/overlay.py @@ -294,7 +294,6 @@ def _create_overlay_dtb( fdt_sw.property("cpus", pack_cpu_ids(instance.resources.cpus)) - fdt_sw.property_u64("memory-base", instance.resources.memory_base) fdt_sw.property_u64("memory-bytes", instance.resources.memory_bytes) if instance.resources.devices: diff --git a/src/kerf/dtc/validator.py b/src/kerf/dtc/validator.py index edca1d7..464f254 100644 --- a/src/kerf/dtc/validator.py +++ b/src/kerf/dtc/validator.py @@ -20,7 +20,7 @@ from pathlib import Path from typing import List, Dict, Optional from ..models import GlobalDeviceTree, ValidationResult, ResourceUsage -from ..resources import get_memory_pool_from_iomem +from ..resources import chunk_containing, get_pool_chunks_from_iomem class MultikernelValidator: @@ -271,36 +271,18 @@ def _validate_hardware_inventory(self, tree: GlobalDeviceTree): self._validate_pool_against_iomem(memory) def _validate_pool_against_iomem(self, memory): - """Compare the pool chunks read back from the kernel with /proc/iomem.""" - iomem_pool = get_memory_pool_from_iomem() - if iomem_pool is not None: - iomem_base, iomem_size = iomem_pool - iomem_end = iomem_base + iomem_size - - if memory.memory_pool_base != iomem_base: - self.errors.append( - f"Hardware inventory: Memory pool base ({hex(memory.memory_pool_base)}) " - f"does not match multikernel pool in /proc/iomem ({hex(iomem_base)})" - ) - - pool_end = memory.memory_pool_base + memory.memory_pool_bytes - if pool_end > iomem_end: - exceeds_by = pool_end - iomem_end - self.errors.append( - f"Hardware inventory: Memory pool extends beyond multikernel reserved pool in /proc/iomem " - f"by {exceeds_by} bytes ({exceeds_by / (1024**3):.2f} GB). " - f"Configured pool: {hex(memory.memory_pool_base)}-{hex(pool_end-1)}, " - f"Reserved pool: {hex(iomem_base)}-{hex(iomem_end-1)}" - ) - - if memory.memory_pool_bytes > iomem_size: - self.errors.append( - f"Hardware inventory: Memory pool size ({memory.memory_pool_bytes} bytes = " - f"{memory.memory_pool_bytes / (1024**3):.2f} GB) exceeds multikernel reserved pool " - f"({iomem_size} bytes = {iomem_size / (1024**3):.2f} GB) in /proc/iomem" - ) - else: + """Every pool chunk the kernel reported must be registered in /proc/iomem.""" + iomem_chunks = set(get_pool_chunks_from_iomem()) + if not iomem_chunks: self.warnings.append("Could not find multikernel memory pool in /proc/iomem") + return + + missing = sorted({(r.base, r.size) for r in memory.regions} - iomem_chunks) + for base, size in missing: + self.errors.append( + f"Hardware inventory: Pool chunk {hex(base)}-{hex(base + size - 1)} " + f"({size / (1024**3):.2f} GB) is not registered in /proc/iomem" + ) def _validate_instances(self, tree: GlobalDeviceTree): """Validate instance definitions.""" @@ -389,34 +371,27 @@ def _validate_cpu_allocation(self, instance, tree: GlobalDeviceTree): def _validate_memory_allocation(self, instance, tree: GlobalDeviceTree): """Validate memory allocation for an instance.""" - memory = tree.hardware.memory instance_memory = instance.resources - memory_start = instance_memory.memory_base - memory_end = memory_start + instance_memory.memory_bytes - if memory_start < memory.memory_pool_base: - error_msg = self._format_error_with_context( - error_type="Memory allocation error", - instance_name=instance.name, - problem=f"Memory base {hex(memory_start)} is before memory pool start", - current_state=f"Spawn pool starts at {hex(memory.memory_pool_base)}", - suggestion=f"Use memory base >= {hex(memory.memory_pool_base)}", - alternative="Adjust memory pool configuration", - pattern=f"memory-base = <{hex(memory_start)}>", - ) - self.errors.append(error_msg) + # A zero base means the instance carries no placement of its own: the + # kernel picks one out of the pool when it creates the instance. + if not memory_start: + return - if memory_end > memory.memory_pool_base + memory.memory_pool_bytes: - pool_end = memory.memory_pool_base + memory.memory_pool_bytes - exceeds_by = memory_end - pool_end + memory_end = memory_start + instance_memory.memory_bytes + chunks = tree.hardware.memory.regions + + if chunks and chunk_containing(tree, memory_start, instance_memory.memory_bytes) is None: + listed = ", ".join(f"{hex(c.base)}-{hex(c.base + c.size - 1)}" for c in chunks) error_msg = self._format_error_with_context( error_type="Memory allocation error", instance_name=instance.name, - problem="Memory region extends beyond memory pool", - current_state=f"Instance memory: {hex(memory_start)}-{hex(memory_end)}, Pool: {hex(memory.memory_pool_base)}-{hex(pool_end)}", - suggestion=f"Reduce memory size by {hex(exceeds_by)} or use different base address", - alternative="Increase memory pool size or adjust memory allocation", + problem="Memory region does not fit in any pool chunk", + current_state=f"Instance memory: {hex(memory_start)}-{hex(memory_end)}, " + f"pool chunks: {listed}", + suggestion="Place the instance inside one chunk or reduce its memory size", + alternative="Grow the pool with 'kerf init --memory=...'", pattern=f"memory-size = <{hex(instance_memory.memory_bytes)}>", ) self.errors.append(error_msg) @@ -433,6 +408,8 @@ def _validate_memory_allocation(self, instance, tree: GlobalDeviceTree): continue other_start = other_instance.resources.memory_base + if not other_start: + continue other_end = other_start + other_instance.resources.memory_bytes if not (memory_end <= other_start or other_end <= memory_start): @@ -444,7 +421,8 @@ def _validate_memory_allocation(self, instance, tree: GlobalDeviceTree): f"Instance {instance.name} and {other_name}: Memory region overlap detected\n" f" {instance.name} memory: {hex(memory_start)} - {hex(memory_end)}\n" f" {other_name} memory: {hex(other_start)} - {hex(other_end)}\n" - f" Overlapping region: {hex(overlap_start)} - {hex(overlap_end)} ({overlap_size} bytes)" + f" Overlapping region: {hex(overlap_start)} - {hex(overlap_end)} " + f"({overlap_size} bytes)" ) def _validate_device_allocation(self, instance, tree: GlobalDeviceTree): diff --git a/src/kerf/resources.py b/src/kerf/resources.py index 06050b9..693bb6d 100644 --- a/src/kerf/resources.py +++ b/src/kerf/resources.py @@ -24,7 +24,7 @@ from pathlib import Path from typing import List, Set, Optional, Tuple -from .models import GlobalDeviceTree +from .models import GlobalDeviceTree, PoolMemoryRegion from .exceptions import ResourceError IOMEM_PATH = "/proc/iomem" @@ -222,63 +222,23 @@ def get_allocated_memory_regions(tree: GlobalDeviceTree) -> List[tuple[int, int] return regions -def find_available_memory_base( - tree: GlobalDeviceTree, size_bytes: int, alignment: int = 0x1000, use_iomem: bool = True -) -> Optional[int]: +def chunk_containing( + tree: GlobalDeviceTree, base: int, size: int +) -> Optional[PoolMemoryRegion]: """ - Find available memory region for allocation. + Find the pool chunk that holds a whole memory region. Args: - tree: GlobalDeviceTree to analyze (for pool boundaries) - size_bytes: Size of memory region needed - alignment: Required alignment (default 4KB) - use_iomem: If True, read actual allocations from /proc/iomem (kernel source of truth). - If False, use allocations from tree (for validation/dry-run). + tree: GlobalDeviceTree carrying the live pool chunks + base: Base address of the region + size: Size of the region in bytes Returns: - Base address for allocation, or None if no space available + The chunk containing the region, or None if no single chunk holds it """ - pool_base = tree.hardware.memory.memory_pool_base - pool_end = tree.hardware.memory.memory_pool_end - - if use_iomem: - allocated_regions = get_allocated_memory_regions_from_iomem() - else: - allocated_regions = get_allocated_memory_regions(tree) - # Sort by base address - allocated_regions.sort() - - # Try to find gap between allocations or at start/end - if not allocated_regions: - # No allocations yet, use start of pool (aligned) - aligned_base = (pool_base + alignment - 1) // alignment * alignment - if aligned_base + size_bytes <= pool_end: - return aligned_base - return None - - # Check gap at start - first_base = allocated_regions[0][0] - aligned_base = (pool_base + alignment - 1) // alignment * alignment - if aligned_base + size_bytes <= first_base: - return aligned_base - - # Check gaps between allocations - for i in range(len(allocated_regions) - 1): - current_end = allocated_regions[i][0] + allocated_regions[i][1] - next_base = allocated_regions[i + 1][0] - - # Align current_end - aligned_base = (current_end + alignment - 1) // alignment * alignment - - if aligned_base + size_bytes <= next_base: - return aligned_base - - # Check gap at end - last_end = allocated_regions[-1][0] + allocated_regions[-1][1] - aligned_base = (last_end + alignment - 1) // alignment * alignment - if aligned_base + size_bytes <= pool_end: - return aligned_base - + for chunk in tree.hardware.memory.regions: + if chunk.base <= base and base + size <= chunk.base + chunk.size: + return chunk return None @@ -348,35 +308,33 @@ def validate_memory_allocation( (for update operations) Raises: - ResourceError: If memory region is invalid or conflicts + ResourceError: If the region is misaligned, leaves the pool chunk that + holds it, or conflicts with another instance """ - pool_base = tree.hardware.memory.memory_pool_base - pool_end = tree.hardware.memory.memory_pool_end - memory_end = memory_base + memory_bytes + if memory_base % 0x1000 != 0: + raise ResourceError(f"Memory base {hex(memory_base)} is not 4KB-aligned") - # Check memory is within pool - if memory_base < pool_base: - raise ResourceError(f"Memory base {hex(memory_base)} is below pool base {hex(pool_base)}") + memory_end = memory_base + memory_bytes - if memory_end > pool_end: + # The pool is a list of chunks, so a region has to sit inside one of them; + # spanning two chunks means spanning the host memory between them. + chunks = tree.hardware.memory.regions + if chunks and chunk_containing(tree, memory_base, memory_bytes) is None: + listed = ", ".join(f"{hex(c.base)}-{hex(c.base + c.size - 1)}" for c in chunks) raise ResourceError( - f"Memory region extends beyond pool: " - f"{hex(memory_base)}-{hex(memory_end)} vs pool end {hex(pool_end)}" + f"Memory region {hex(memory_base)}-{hex(memory_end)} does not fit in " + f"any pool chunk ({listed})" ) - # Check alignment (4KB) - if memory_base % 0x1000 != 0: - raise ResourceError(f"Memory base {hex(memory_base)} is not 4KB-aligned") - - # Check for overlaps with other instances for instance in tree.instances.values(): if instance.name == exclude_instance: continue inst_base = instance.resources.memory_base + if not inst_base: + continue inst_end = inst_base + instance.resources.memory_bytes - # Check for overlap if not (memory_end <= inst_base or memory_base >= inst_end): raise ResourceError( f"Memory region {hex(memory_base)}-{hex(memory_end)} " diff --git a/src/kerf/runtime.py b/src/kerf/runtime.py index 081709a..078b6ac 100644 --- a/src/kerf/runtime.py +++ b/src/kerf/runtime.py @@ -69,18 +69,13 @@ def create_instance(current: GlobalDeviceTree) -> GlobalDeviceTree: # Validate resources (against baseline) validate_cpu_allocation(modified, cpus) - # Find memory base - memory_base = find_available_memory_base(modified, memory) - if not memory_base: - raise ResourceError("No memory available") - - # Create instance + # Create instance; the kernel places its memory in the pool instance = Instance( name=name, id=find_next_instance_id(modified), resources=InstanceResources( cpus=cpus, - memory_base=memory_base, + memory_base=0, memory_bytes=memory, devices=[] ) diff --git a/src/kerf/update/main.py b/src/kerf/update/main.py index c51a013..c65e94b 100644 --- a/src/kerf/update/main.py +++ b/src/kerf/update/main.py @@ -32,7 +32,7 @@ from ..create.main import parse_cpu_spec, parse_memory_base, parse_memory_spec from ..exceptions import KernelInterfaceError, ParseError, ResourceError, ValidationError from ..resources import ( - find_available_memory_base, + chunk_containing, validate_cpu_allocation, validate_memory_allocation, ) @@ -281,35 +281,23 @@ def update_instance_operation(current): if memory_bytes is not None: if memory_base_addr is None: - # Keep the same base address and extend/shrink in place old_base = existing_instance.resources.memory_base old_size = existing_instance.resources.memory_bytes + # The overlay names an existing range, so an instance can + # only grow into the chunk it already sits in. if memory_bytes > old_size: - # Growing: validate the extension region doesn't overlap - extension_base = old_base + old_size - extension_size = memory_bytes - old_size - try: - validate_memory_allocation( - modified, extension_base, extension_size, - exclude_instance=instance_node_name + if chunk_containing(modified, old_base, memory_bytes) is None: + raise ResourceError( + f"Cannot grow instance '{name}' to {memory_bytes} bytes: " + f"the extension leaves the pool chunk holding " + f"{hex(old_base)}-{hex(old_base + old_size - 1)}" ) - memory_base_addr = old_base - except (ResourceError,) as exc: - # Extension conflicts, find a completely new region - found_base = find_available_memory_base(modified, memory_bytes) - if found_base is None: - raise ResourceError( - f"No available memory region found for {memory_bytes} bytes. " - "Try specifying --memory-base or reduce memory size." - ) from exc - memory_base_addr = found_base - elif memory_bytes < old_size: - # Shrinking: always keep the same base - memory_base_addr = old_base - else: - # Same size, no change - memory_base_addr = old_base + validate_memory_allocation( + modified, old_base + old_size, memory_bytes - old_size, + exclude_instance=instance_node_name + ) + memory_base_addr = old_base else: validate_memory_allocation( modified, memory_base_addr, memory_bytes, exclude_instance=instance_node_name diff --git a/tests/test_pool_overlay.py b/tests/test_pool_overlay.py index 6f397dc..7564c97 100644 --- a/tests/test_pool_overlay.py +++ b/tests/test_pool_overlay.py @@ -142,6 +142,11 @@ def test_create_overlay_targets_the_instance_namespace(sample_hardware, sample_i assert fdt.getprop(fdt.path_offset("/fragment@0"), "target-path").as_str() == "/instances" create = fdt.subnode_offset(ov, "instance-create") assert fdt.getprop(create, "instance-name").as_str() == "database" + # The kernel places instance memory itself, so the request names no base. + resources = fdt.subnode_offset(create, "resources") + assert fdt.getprop(resources, "memory-base", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + assert fdt.getprop(resources, "memory-bytes").as_uint64() == \ + sample_instances["database"].resources.memory_bytes for offset in _walk(fdt): assert fdt.getprop(offset, "mk,instance", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING diff --git a/tests/test_resources.py b/tests/test_resources.py index 6efb193..2480a52 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -26,7 +26,7 @@ get_pool_allocated_bytes, get_pool_chunks_from_iomem, get_busy_chunks_from_iomem, - find_available_memory_base, + chunk_containing, validate_cpu_allocation, validate_memory_allocation, find_next_instance_id, @@ -102,38 +102,25 @@ def test_get_allocated_memory_regions(self, sample_tree): assert 0x80000000 in bases # web-server assert 0x100000000 in bases # database - def test_find_available_memory_base_empty_pool(self, sample_hardware): - """Test finding memory base in empty pool.""" - from kerf.models import GlobalDeviceTree - - # Create tree with no instances - tree = GlobalDeviceTree(hardware=sample_hardware, instances={}, device_references={}) - - # Request 1GB - size = 1024**3 - base = find_available_memory_base(tree, size, use_iomem=False) + def test_chunk_containing_finds_the_holding_chunk(self, sample_tree): + """A region inside a chunk resolves to that chunk.""" + chunk = chunk_containing(sample_tree, 0x100000000, 1024**3) - # Should get start of pool (aligned) - assert base == sample_hardware.memory.memory_pool_base + assert chunk is sample_tree.hardware.memory.regions[0] - def test_find_available_memory_base_with_allocations(self, sample_tree): - """Test finding memory base with existing allocations.""" - # Request 1GB after existing allocations - size = 1024**3 - base = find_available_memory_base(sample_tree, size, use_iomem=False) + def test_chunk_containing_rejects_a_region_spanning_two_chunks(self, sample_hardware): + """Host memory between two chunks is not part of the pool.""" + from kerf.models import GlobalDeviceTree, PoolMemoryRegion - # Should find a gap or append at end - assert base is not None - assert base >= sample_tree.hardware.memory.memory_pool_base - - def test_find_available_memory_base_no_space(self, sample_tree): - """Test finding memory base when no space available.""" - # Request more memory than available in pool - size = 100 * 1024**3 # 100GB - way more than pool size - base = find_available_memory_base(sample_tree, size, use_iomem=False) + sample_hardware.memory.regions = [ + PoolMemoryRegion(base=0x100000000, size=1024**3, node=0), + PoolMemoryRegion(base=0x200000000, size=1024**3, node=1), + ] + tree = GlobalDeviceTree(hardware=sample_hardware, instances={}, device_references={}) - # Should return None - assert base is None + assert chunk_containing(tree, 0x100000000, 1024**3) is not None + assert chunk_containing(tree, 0x200000000, 1024**3) is not None + assert chunk_containing(tree, 0x1C0000000, 1024**3) is None def test_validate_memory_allocation_success(self, sample_tree): """Test successful memory allocation validation.""" @@ -155,12 +142,12 @@ def test_validate_memory_allocation_overlap(self, sample_tree): validate_memory_allocation(sample_tree, memory_base, memory_bytes) def test_validate_memory_allocation_out_of_pool(self, sample_tree): - """Test memory allocation outside pool.""" - # Use base before pool - memory_base = 0x10000000 # Below pool base + """Test memory allocation outside every pool chunk.""" + # Use base before the only chunk + memory_base = 0x10000000 memory_bytes = 1024**3 - with pytest.raises(ResourceError, match="below pool base"): + with pytest.raises(ResourceError, match="does not fit in any pool chunk"): validate_memory_allocation(sample_tree, memory_base, memory_bytes) def test_validate_memory_allocation_misaligned(self, sample_tree): diff --git a/tests/test_validator.py b/tests/test_validator.py index 76c9130..a6df621 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -16,6 +16,7 @@ Tests for kerf validator. """ +from kerf.dtc import validator as validator_module from kerf.dtc.validator import MultikernelValidator @@ -386,3 +387,90 @@ def test_validate_spread_affinity(self, sample_hardware): assert any( "Spread" in warning and "single NUMA node" in warning for warning in result.warnings ) + + +class TestPoolChunks: + """Test the pool read-back against /proc/iomem.""" + + @staticmethod + def _two_chunk_tree(sample_hardware): + from kerf.models import GlobalDeviceTree, PoolMemoryRegion + + sample_hardware.memory.regions = [ + PoolMemoryRegion(base=0x100000000, size=1024**3, node=0), + PoolMemoryRegion(base=0x200000000, size=512 * 1024**2, node=1), + ] + return GlobalDeviceTree(hardware=sample_hardware, instances={}, device_references={}) + + def test_every_chunk_registered_in_iomem_passes(self, sample_hardware, monkeypatch): + """A pool of several non-contiguous chunks is valid.""" + tree = self._two_chunk_tree(sample_hardware) + monkeypatch.setattr( + validator_module, "get_pool_chunks_from_iomem", + lambda: [(0x100000000, 1024**3), (0x200000000, 512 * 1024**2)], + ) + + result = MultikernelValidator().validate(tree) + + assert result.is_valid, result.errors + + def test_chunk_missing_from_iomem_fails(self, sample_hardware, monkeypatch): + """A chunk the kernel reports but /proc/iomem does not is an error.""" + tree = self._two_chunk_tree(sample_hardware) + monkeypatch.setattr( + validator_module, "get_pool_chunks_from_iomem", + lambda: [(0x100000000, 1024**3)], + ) + + result = MultikernelValidator().validate(tree) + + assert not result.is_valid + assert any("0x200000000" in error and "not registered in /proc/iomem" in error + for error in result.errors) + + def test_instance_inside_the_second_chunk_is_valid(self, sample_hardware, monkeypatch): + """An instance placed in a later chunk is not out of pool.""" + from kerf.models import Instance, InstanceResources + + tree = self._two_chunk_tree(sample_hardware) + tree.instances = { + "app1": Instance( + name="app1", + id=1, + resources=InstanceResources( + cpus=[4], memory_base=0x200000000, memory_bytes=128 * 1024**2, devices=[] + ), + ) + } + monkeypatch.setattr( + validator_module, "get_pool_chunks_from_iomem", + lambda: [(0x100000000, 1024**3), (0x200000000, 512 * 1024**2)], + ) + + result = MultikernelValidator().validate(tree) + + assert result.is_valid, result.errors + + def test_instance_spanning_two_chunks_fails(self, sample_hardware, monkeypatch): + """A region crossing the gap between chunks covers host memory.""" + from kerf.models import Instance, InstanceResources + + tree = self._two_chunk_tree(sample_hardware) + tree.instances = { + "app1": Instance( + name="app1", + id=1, + resources=InstanceResources( + cpus=[4], memory_base=0x1C0000000, memory_bytes=2 * 1024**3, devices=[] + ), + ) + } + monkeypatch.setattr( + validator_module, "get_pool_chunks_from_iomem", + lambda: [(0x100000000, 1024**3), (0x200000000, 512 * 1024**2)], + ) + + result = MultikernelValidator().validate(tree) + + assert not result.is_valid + assert any("does not fit in any pool chunk" in error for error in result.errors) From a2d36c873bbdd44c247e7594aabb59fe13388453 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 13:15:40 -0700 Subject: [PATCH 14/23] parser: Accept a live pool that holds no memory A transaction can hand back every chunk and keep the pool CPUs, so a read-back with cpus-available but no memory@ node describes a real pool, not a broken one; rejecting it made kerf init fall back to writing a baseline, which the kernel refuses while it owns resources. Accept that shape as a pool with no memory while still requiring a request baseline to name its memory, since nothing else says how much pool to build. Signed-off-by: Cong Wang --- src/kerf/dtc/parser.py | 18 +++++++++++++++--- tests/test_init_reconcile.py | 18 ++++++++++++++++++ tests/test_parser_pool.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/kerf/dtc/parser.py b/src/kerf/dtc/parser.py index 0eac0ab..b806615 100644 --- a/src/kerf/dtc/parser.py +++ b/src/kerf/dtc/parser.py @@ -49,6 +49,9 @@ "expected memory@N { size; numa-node-id; }" ) +# A live pool always publishes cpus-available, and may legitimately hold no +# memory at all: a transaction that gave back every chunk but kept the CPUs. + class DeviceTreeParser: """Parser for multikernel device trees.""" @@ -302,7 +305,8 @@ def _parse_memory_allocation(self, resources_node: int) -> MemoryAllocation: raise ParseError(_LEGACY_MEMORY_ERROR) if not regions and not requested: - raise ParseError(_NO_MEMORY_ERROR) + if self._optional_prop(resources_node, 'cpus-available') is None: + raise ParseError(_NO_MEMORY_ERROR) total_bytes = sum(r.size for r in regions) or sum(requested.values()) @@ -821,6 +825,12 @@ def _parse_cpus_from_dts(self, dts_content: str) -> CPUAllocation: raise ParseError("Missing 'cpus' property in /resources") available = [int(x.strip()) for x in cpus_match.group(1).split()] + + free_match = re.search(r'cpus-available\s*=\s*<([^>]+)>', resources_text) + available_free = None + if free_match: + available_free = [int(x.strip()) for x in free_match.group(1).split()] + if available: total = max(available) + 1 else: @@ -834,7 +844,8 @@ def _parse_cpus_from_dts(self, dts_content: str) -> CPUAllocation: total=total, host_reserved=host_reserved, available=available, - topology=topology + topology=topology, + available_free=available_free ) def _split_node_body(self, body: str) -> Tuple[str, List[Tuple[str, str]]]: @@ -907,7 +918,8 @@ def _parse_memory_from_dts(self, dts_content: str) -> MemoryAllocation: raise ParseError(f"{name}: expected 'reg' (existing chunk) or 'size' (request)") if not regions and not requested: - raise ParseError(_NO_MEMORY_ERROR) + if not re.search(r'cpus-available\s*=', own_properties): + raise ParseError(_NO_MEMORY_ERROR) total_bytes = sum(r.size for r in regions) or sum(requested.values()) diff --git a/tests/test_init_reconcile.py b/tests/test_init_reconcile.py index aaee41f..bf6307a 100644 --- a/tests/test_init_reconcile.py +++ b/tests/test_init_reconcile.py @@ -208,6 +208,24 @@ def test_teardown_refuses_while_instances_exist(monkeypatch): assert not manager.applied +def test_cpu_only_pool_grows_memory(): + # A pool that kept its CPUs but gave back every chunk is live, and the + # request only has to add the memory back. + manager = FakeManager() + current = _tree([1, 2, 3], [], {}, available_free=[1, 2, 3]) + requested = _tree([1, 2, 3], [], {-1: GB // 2}) + baseline_mgr = FakeBaselineManager( + live=_tree([1, 2, 3], [PoolMemoryRegion(0x1_0000_0000, GB // 2, 0)], {})) + + diff = reconcile_pool(current, requested, set(), False, manager, baseline_mgr) + + assert diff.memory_to_pool == [(-1, GB // 2)] + assert diff.cpus_to_pool == [] and diff.cpus_to_host == [] + assert diff.memory_to_host == [] + assert len(manager.applied) == 1 + assert not baseline_mgr.written + + def test_pool_cpus_stay_valid_apic_ids(monkeypatch): # A CPU in the pool is gone from /proc/cpuinfo, but re-init must still name it. monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0}) diff --git a/tests/test_parser_pool.py b/tests/test_parser_pool.py index d4a6f91..1917a3f 100644 --- a/tests/test_parser_pool.py +++ b/tests/test_parser_pool.py @@ -209,6 +209,17 @@ def test_parse_dts_legacy_memory_base_bytes_rejected(): }; """ +_DTS_LIVE_NO_MEMORY = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + cpus-available = <5>; + }; +}; +""" + _DTS_NO_MEMORY = """ /multikernel-v1/; @@ -239,6 +250,25 @@ def build(sw): DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) +def test_parse_dts_live_pool_without_memory_accepted(): + tree = DeviceTreeParser().parse_dts(_DTS_LIVE_NO_MEMORY) + assert tree.hardware.memory.regions == [] + assert tree.hardware.memory.requested == {} + assert tree.hardware.memory.total_bytes == 0 + assert tree.hardware.cpus.available_free == [5] + + +def test_parse_dtb_live_pool_without_memory_accepted(): + def build(sw): + sw.property("cpus-available", struct.pack(">Q", 5)) + + tree = DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + assert tree.hardware.memory.regions == [] + assert tree.hardware.memory.requested == {} + assert tree.hardware.memory.total_bytes == 0 + assert tree.hardware.cpus.available_free == [5] + + def test_parse_dtb_legacy_memory_base_alone_rejected(): def build(sw): sw.property_u64("memory-base", 0x80000000) From 18ddf57a809ae1afaef1dc0366402eeab22b56bd Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 13:16:52 -0700 Subject: [PATCH 15/23] init: Say why the pool did not shrink Only whole idle chunks go back to the host, so a request smaller than every chunk leaves the pool as it is and the diff comes out empty, which looked like success with no explanation. Report the surplus on the paths that move nothing, the empty diff and --dry-run, the way the path that applies a transaction already does, and give sizes coming from --input the same page-alignment check as --memory since the kernel rejects the rest either way. Signed-off-by: Cong Wang --- src/kerf/init/main.py | 47 +++++++++++++++++++++++++--------- tests/test_init_memory_spec.py | 13 +++++++++- tests/test_init_reconcile.py | 26 ++++++++++++++++++- tests/test_pool_diff.py | 12 +++++++++ 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/src/kerf/init/main.py b/src/kerf/init/main.py index a1c2afb..1ecb759 100644 --- a/src/kerf/init/main.py +++ b/src/kerf/init/main.py @@ -48,7 +48,7 @@ HardwareInventory, MemoryAllocation, ) -from ..pool_diff import PoolDiff, compute_pool_diff +from ..pool_diff import ANY_NODE, PoolDiff, compute_pool_diff from ..resources import get_busy_chunks_from_iomem from ..runtime import DeviceTreeManager @@ -359,6 +359,26 @@ def get_valid_apic_ids_from_system() -> Optional[set]: PAGE_SIZE = 4096 +def validate_memory_request(requested: Dict[int, int]) -> None: + """ + Reject pool sizes the kernel cannot honour, however they were asked for. + + Args: + requested: Mapping of NUMA node id (ANY_NODE for any node) to size + + Raises: + ValueError: If a size is zero, negative or not page aligned + """ + for node, size in sorted(requested.items()): + where = "any node" if node == ANY_NODE else f"node {node}" + if size <= 0: + raise ValueError(f"memory size for {where} must be greater than zero") + if size % PAGE_SIZE: + raise ValueError( + f"memory size for {where} must be a multiple of {PAGE_SIZE} bytes" + ) + + def parse_memory_request(spec: str) -> Dict[int, int]: """ Parse a pool memory request into per-NUMA-node sizes. @@ -388,19 +408,14 @@ def parse_memory_request(spec: str) -> Dict[int, int]: elif part.lower().startswith("node"): raise ValueError(f"invalid NUMA node spec '{part}' (expected nodeN:SIZE)") else: - node, size = -1, parse_memory_spec(part) - if size <= 0: - raise ValueError(f"memory size in '{part}' must be greater than zero") - if size % PAGE_SIZE: - raise ValueError( - f"memory size in '{part}' must be a multiple of {PAGE_SIZE} bytes" - ) + node, size = ANY_NODE, parse_memory_spec(part) if node in requested: raise ValueError(f"node {node} specified twice") requested[node] = size - if -1 in requested and len(requested) > 1: + if ANY_NODE in requested and len(requested) > 1: raise ValueError("cannot mix a plain size with nodeN: sizes") + validate_memory_request(requested) return requested @@ -623,7 +638,7 @@ def line(label, items): line("CPUs to pool", diff.cpus_to_pool) line("CPUs to host", diff.cpus_to_host) line("Memory to pool", [ - f"{size >> 20} MB" + (f" on node {node}" if node >= 0 else "") + f"{size >> 20} MB" + ("" if node == ANY_NODE else f" on node {node}") for node, size in diff.memory_to_pool ]) line("Memory to host", [f"{hex(r.base)} ({r.size >> 20} MB)" for r in diff.memory_to_host]) @@ -634,12 +649,12 @@ def line(label, items): def _report_shortfall(live: GlobalDeviceTree, requested: GlobalDeviceTree) -> None: """Warn when the kernel could not shrink the pool all the way down.""" for node, want in requested.hardware.memory.requested.items(): - if node < 0: + if node == ANY_NODE: have = live.hardware.memory.memory_pool_bytes else: have = live.hardware.memory.bytes_on_node(node) if have > want: - where = node if node >= 0 else "any" + where = "any" if node == ANY_NODE else node click.echo( f"Note: node {where} still holds {(have - want) >> 20} MB more than " "requested; only whole idle chunks can be returned", @@ -694,9 +709,11 @@ def reconcile_pool( _print_diff(diff) if diff.is_empty(): click.echo("✓ Pool already matches the request; nothing to do") + _report_shortfall(current, requested) return diff if dry_run: click.echo("Would apply the changes above (dry-run)") + _report_shortfall(current, requested) return diff if request_is_empty(requested): @@ -853,6 +870,12 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: click.echo(f"Error: Unsupported input format: {input_path.suffix}", err=True) click.echo("Supported formats: .dts, .dtb", err=True) sys.exit(2) + + try: + validate_memory_request(tree.hardware.memory.requested) + except ValueError as e: + click.echo(f"Error: {input}: {e}", err=True) + sys.exit(2) else: # Build from command line arguments try: diff --git a/tests/test_init_memory_spec.py b/tests/test_init_memory_spec.py index 5537732..e90afb1 100644 --- a/tests/test_init_memory_spec.py +++ b/tests/test_init_memory_spec.py @@ -16,7 +16,7 @@ import pytest -from kerf.init.main import parse_memory_request +from kerf.init.main import parse_memory_request, validate_memory_request GB = 1 << 30 @@ -39,3 +39,14 @@ def test_invalid_specs(spec): def test_sizes_must_be_positive_and_page_aligned(spec): with pytest.raises(ValueError): parse_memory_request(spec) + + +@pytest.mark.parametrize("requested", [{-1: 0}, {0: -4096}, {0: 4097}, {1: 5000}]) +def test_input_file_sizes_get_the_same_check(requested): + # A DTS/DTB request reaches the kernel without going through --memory. + with pytest.raises(ValueError): + validate_memory_request(requested) + + +def test_valid_input_file_sizes_pass(): + validate_memory_request({0: GB, 1: 512 << 20}) diff --git a/tests/test_init_reconcile.py b/tests/test_init_reconcile.py index bf6307a..50a29db 100644 --- a/tests/test_init_reconcile.py +++ b/tests/test_init_reconcile.py @@ -208,6 +208,30 @@ def test_teardown_refuses_while_instances_exist(monkeypatch): assert not manager.applied +def test_unshrinkable_surplus_is_reported(capsys): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + current = _tree([4, 5], [PoolMemoryRegion(0x1_0000_0000, 2 * GB, 0)], {}) + requested = _tree([4, 5], [], {0: GB}) + + diff = reconcile_pool(current, requested, set(), False, manager, baseline_mgr) + + assert diff.is_empty() + assert not manager.applied + err = capsys.readouterr().err + assert "node 0 still holds 1024 MB more than requested" in err + assert "only whole idle chunks can be returned" in err + + +def test_dry_run_reports_the_surplus_too(capsys): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + current = _tree([4], [PoolMemoryRegion(0x1_0000_0000, 2 * GB, 0)], {}) + requested = _tree([4, 5], [], {0: GB}) + + reconcile_pool(current, requested, set(), True, manager, baseline_mgr) + + assert "still holds 1024 MB more than requested" in capsys.readouterr().err + + def test_cpu_only_pool_grows_memory(): # A pool that kept its CPUs but gave back every chunk is live, and the # request only has to add the memory back. @@ -259,7 +283,7 @@ def test_teardown_tree_reserves_pool_cpus(monkeypatch): tree = main.build_teardown_tree(pool_cpus={1, 2, 3}) assert tree.hardware.cpus.host_reserved == [0, 1, 2, 3] - assert tree.hardware.cpus.available == [] + assert not tree.hardware.cpus.available assert tree.hardware.cpus.total == 4 diff --git a/tests/test_pool_diff.py b/tests/test_pool_diff.py index 243d7c9..263b00c 100644 --- a/tests/test_pool_diff.py +++ b/tests/test_pool_diff.py @@ -90,3 +90,15 @@ def test_shrink_prefers_larger_idle_chunks(): d = compute_pool_diff(cur, req) assert d.memory_to_host == [big] assert d.memory_to_pool == [] + + +def test_surplus_smaller_than_every_chunk_stays_in_the_pool(): + # Only whole chunks go back to the host, so a 1GB pool asked to shrink to + # 512MB keeps its chunk and the caller has to be told. + chunk = PoolMemoryRegion(0x1_0000_0000, GB, 0) + cur = _tree([4], regions=[chunk]) + req = _tree([4], requested={0: GB // 2}) + d = compute_pool_diff(cur, req) + assert d.memory_to_host == [] + assert d.memory_to_pool == [] + assert d.is_empty() From 1496e9dd9c98f206314c0d9a3b70efcb8f7e7734 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 13:17:15 -0700 Subject: [PATCH 16/23] resources: Take the free CPU list from the kernel The root read-back publishes pool membership in cpus and the free members in cpus-available, and carries no instances section. Deriving free CPUs by subtracting the tree's instances from membership therefore offered CPUs already lent to a running instance, and kerf create only found out when the kernel refused the transaction; prefer cpus-available whenever the tree carries it. Signed-off-by: Cong Wang --- src/kerf/resources.py | 16 +++++++++------- tests/test_resources.py | 13 +++++++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/kerf/resources.py b/src/kerf/resources.py index 693bb6d..43cbdd6 100644 --- a/src/kerf/resources.py +++ b/src/kerf/resources.py @@ -149,21 +149,23 @@ def get_available_cpus(tree: GlobalDeviceTree) -> Set[int]: """ Get set of CPUs available for allocation (not allocated to any instance). + The kernel's own free list wins when the tree carries one: the root + read-back has no instances section, so deriving free CPUs from pool + membership alone would hand out CPUs that are already lent out. + Args: tree: GlobalDeviceTree to analyze Returns: Set of available CPU IDs """ - # Get all CPUs in the available pool - available = set(tree.hardware.cpus.available) + allocated = get_allocated_cpus(tree) - # Subtract CPUs allocated to instances - allocated = set() - for instance in tree.instances.values(): - allocated.update(instance.resources.cpus) + free = tree.hardware.cpus.available_free + if free is not None: + return set(free) - allocated - return available - allocated + return set(tree.hardware.cpus.available) - allocated def get_allocated_cpus(tree: GlobalDeviceTree) -> Set[int]: diff --git a/tests/test_resources.py b/tests/test_resources.py index 2480a52..8faf825 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -46,6 +46,19 @@ def test_get_available_cpus(self, sample_tree): # So 16-31 should be available assert available == set(range(16, 32)) + def test_get_available_cpus_trusts_the_kernel_free_list(self, sample_tree): + """A read-back lists no instances, so its free list is the only truth.""" + sample_tree.hardware.cpus.available_free = [20, 21, 22] + sample_tree.instances = {} + + assert get_available_cpus(sample_tree) == {20, 21, 22} + + def test_get_available_cpus_free_list_minus_tree_instances(self, sample_tree): + """When the tree does list instances, their CPUs are not free.""" + sample_tree.hardware.cpus.available_free = [4, 5, 20] + + assert get_available_cpus(sample_tree) == {20} + def test_get_allocated_cpus(self, sample_tree): """Test getting allocated CPUs.""" allocated = get_allocated_cpus(sample_tree) From 096017be3358b579b4e1ee00ce90972947f87135 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 13:19:37 -0700 Subject: [PATCH 17/23] kerf: Clean up around the pool interface Nothing here changes what the kernel is asked to do: name the pool transaction file's contents for what they are, the target path of the overlay's first fragment rather than an instance; make the device tree lock public so callers stop reaching into a private method with a pylint waiver; drop the DTB writers and the u64 reader that nothing calls, and a hardware check whose message contradicted its condition; spell the any-node NUMA id as pool_diff.ANY_NODE; say "No memory pool configured" after a teardown instead of printing nothing; and name the example's device NUMA property numa-node-id as everywhere else. Signed-off-by: Cong Wang --- examples/numa_topology.dts | 8 ++-- src/kerf/dtc/extractor.py | 75 +----------------------------------- src/kerf/dtc/overlay.py | 4 +- src/kerf/dtc/parser.py | 12 ++---- src/kerf/dtc/validator.py | 3 -- src/kerf/init/main.py | 2 +- src/kerf/runtime.py | 12 +++--- src/kerf/show/main.py | 40 ++++++++++++------- src/kerf/update/main.py | 2 +- tests/test_init_reconcile.py | 2 +- tests/test_runtime.py | 4 +- tests/test_show.py | 11 ++++++ 12 files changed, 60 insertions(+), 115 deletions(-) diff --git a/examples/numa_topology.dts b/examples/numa_topology.dts index a396824..503d438 100644 --- a/examples/numa_topology.dts +++ b/examples/numa_topology.dts @@ -82,7 +82,7 @@ eth0: ethernet@0 { compatible = "intel,i40e"; pci-id = "0000:01:00.0"; - numa-node = <0>; + numa-node-id = <0>; sriov-vfs = <8>; host-reserved-vf = <0>; available-vfs = <1 2 3 4 5 6 7>; @@ -91,7 +91,7 @@ eth1: ethernet@1 { compatible = "intel,i40e"; pci-id = "0000:02:00.0"; - numa-node = <2>; + numa-node-id = <2>; sriov-vfs = <8>; host-reserved-vf = <0>; available-vfs = <1 2 3 4 5 6 7>; @@ -100,7 +100,7 @@ nvme0: storage@0 { compatible = "nvme"; pci-id = "0000:03:00.0"; - numa-node = <0>; + numa-node-id = <0>; namespaces = <4>; host-reserved-ns = <1>; available-ns = <2 3 4>; @@ -109,7 +109,7 @@ nvme1: storage@1 { compatible = "nvme"; pci-id = "0000:04:00.0"; - numa-node = <2>; + numa-node-id = <2>; namespaces = <4>; host-reserved-ns = <1>; available-ns = <2 3 4>; diff --git a/src/kerf/dtc/extractor.py b/src/kerf/dtc/extractor.py index b4d0fdf..f45055b 100644 --- a/src/kerf/dtc/extractor.py +++ b/src/kerf/dtc/extractor.py @@ -18,6 +18,7 @@ import libfdt from ..models import GlobalDeviceTree, Instance +from ..pool_diff import ANY_NODE from .cells import pack_cpu_ids @@ -130,7 +131,7 @@ def _add_memory_properties_sw(self, fdt_sw, memory): for idx, (node, size) in enumerate(entries): fdt_sw.begin_node(f"memory@{idx}") fdt_sw.property_u64("size", size) - if node >= 0: + if node != ANY_NODE: fdt_sw.property_u32("numa-node-id", node) fdt_sw.end_node() @@ -249,78 +250,6 @@ def _add_device_references_sw(self, fdt_sw, device_references): fdt_sw.end_node() - def _add_resources_section(self, parent_offset: int, tree: GlobalDeviceTree): - """Add resources section to DTB.""" - resources_offset = self.fdt.add_subnode(parent_offset, "resources") - - # Add CPU information - self._add_cpu_section(resources_offset, tree.hardware.cpus) - - # Add memory information - self._add_memory_section(resources_offset, tree.hardware.memory) - - # Add device information - self._add_devices_section(resources_offset, tree.hardware.devices) - - def _add_cpu_section(self, parent_offset: int, cpus): - """Add CPU section to DTB.""" - cpus_offset = self.fdt.add_subnode(parent_offset, "cpus") - self.fdt.setprop_u32(cpus_offset, "total", cpus.total) - - self.fdt.setprop(cpus_offset, "host-reserved", pack_cpu_ids(cpus.host_reserved)) - self.fdt.setprop(cpus_offset, "available", pack_cpu_ids(cpus.available)) - - def _add_memory_section(self, parent_offset: int, memory): - """Add memory section to DTB.""" - memory_offset = self.fdt.add_subnode(parent_offset, "memory") - self.fdt.setprop_u64(memory_offset, "total-bytes", memory.total_bytes) - self.fdt.setprop_u64(memory_offset, "host-reserved-bytes", memory.host_reserved_bytes) - self.fdt.setprop_u64(memory_offset, "memory-pool-base", memory.memory_pool_base) - self.fdt.setprop_u64(memory_offset, "memory-pool-bytes", memory.memory_pool_bytes) - - def _add_devices_section(self, parent_offset: int, devices): - """Add devices section to DTB.""" - devices_offset = self.fdt.add_subnode(parent_offset, "devices") - - for name, device_info in devices.items(): - device_offset = self.fdt.add_subnode(devices_offset, name) - self.fdt.setprop_str(device_offset, "compatible", device_info.compatible) - - if device_info.pci_id: - self.fdt.setprop_str(device_offset, "pci-id", device_info.pci_id) - - if device_info.sriov_vfs is not None: - self.fdt.setprop_u32(device_offset, "sriov-vfs", device_info.sriov_vfs) - - if device_info.host_reserved_vf is not None: - self.fdt.setprop_u32( - device_offset, "host-reserved-vf", device_info.host_reserved_vf - ) - - if device_info.available_vfs: - import struct - - vfs_data = struct.pack( - ">" + "I" * len(device_info.available_vfs), *device_info.available_vfs - ) - self.fdt.setprop(device_offset, "available-vfs", vfs_data) - - if device_info.namespaces is not None: - self.fdt.setprop_u32(device_offset, "namespaces", device_info.namespaces) - - if device_info.host_reserved_ns is not None: - self.fdt.setprop_u32( - device_offset, "host-reserved-ns", device_info.host_reserved_ns - ) - - if device_info.available_ns: - import struct - - ns_data = struct.pack( - ">" + "I" * len(device_info.available_ns), *device_info.available_ns - ) - self.fdt.setprop(device_offset, "available-ns", ns_data) - def _add_instances_section(self, parent_offset: int, tree: GlobalDeviceTree): """Add instances section to DTB.""" instances_offset = self.fdt.add_subnode(parent_offset, "instances") diff --git a/src/kerf/dtc/overlay.py b/src/kerf/dtc/overlay.py index 1cc1166..d9dbc3f 100644 --- a/src/kerf/dtc/overlay.py +++ b/src/kerf/dtc/overlay.py @@ -25,7 +25,7 @@ import libfdt from ..models import GlobalDeviceTree -from ..pool_diff import PoolDiff +from ..pool_diff import ANY_NODE, PoolDiff from .cells import pack_cpu_id, pack_cpu_ids Range = Optional[Tuple[int, int]] @@ -202,7 +202,7 @@ def generate_pool_overlay(self, diff: PoolDiff) -> bytes: for idx, (node, size) in enumerate(diff.memory_to_pool): fdt_sw.begin_node(f"memory@{idx}") fdt_sw.property_u64("size", size) - if node >= 0: + if node != ANY_NODE: fdt_sw.property_u32("numa-node-id", node) fdt_sw.end_node() fdt_sw.end_node() diff --git a/src/kerf/dtc/parser.py b/src/kerf/dtc/parser.py index b806615..0c95131 100644 --- a/src/kerf/dtc/parser.py +++ b/src/kerf/dtc/parser.py @@ -23,6 +23,7 @@ import libfdt from ..exceptions import ParseError +from ..pool_diff import ANY_NODE from .cells import unpack_cpu_ids from ..models import ( CPUAllocation, @@ -269,13 +270,6 @@ def _optional_u32(self, node: int, name: str, default: int) -> int: return default return prop.as_uint32() - def _optional_u64(self, node: int, name: str) -> Optional[int]: - """Return a u64 property, or None when the node does not carry it.""" - prop = self._optional_prop(node, name) - if prop is None: - return None - return prop.as_uint64() - def _parse_memory_allocation(self, resources_node: int) -> MemoryAllocation: """Parse pool chunks and per-node memory requests from resources node.""" regions = [] @@ -285,7 +279,7 @@ def _parse_memory_allocation(self, resources_node: int) -> MemoryAllocation: name = self.fdt.get_name(node) if not name.startswith('memory@'): continue - node_id = self._optional_u32(node, 'numa-node-id', -1) + node_id = self._optional_u32(node, 'numa-node-id', ANY_NODE) reg = self._optional_prop(node, 'reg') size_prop = self._optional_prop(node, 'size') if reg is not None: @@ -897,7 +891,7 @@ def _parse_memory_from_dts(self, dts_content: str) -> MemoryAllocation: for name, body in children: if not name.startswith('memory@'): continue - node_id = -1 + node_id = ANY_NODE node_match = re.search(r'numa-node-id\s*=\s*<\s*([^>\s]+)\s*>', body) if node_match: node_id = int(node_match.group(1), 0) diff --git a/src/kerf/dtc/validator.py b/src/kerf/dtc/validator.py index 464f254..d5e46d3 100644 --- a/src/kerf/dtc/validator.py +++ b/src/kerf/dtc/validator.py @@ -254,9 +254,6 @@ def _validate_hardware_inventory(self, tree: GlobalDeviceTree): ) memory = tree.hardware.memory - if memory.total_bytes < 0: - self.errors.append("Hardware inventory: Total memory must be positive") - if memory.memory_pool_bytes <= 0: self.errors.append("Hardware inventory: Spawn pool size must be positive") diff --git a/src/kerf/init/main.py b/src/kerf/init/main.py index 1ecb759..de562bc 100644 --- a/src/kerf/init/main.py +++ b/src/kerf/init/main.py @@ -724,7 +724,7 @@ def reconcile_pool( f"delete instances {', '.join(held)} first" ) - with manager._acquire_lock(): # pylint: disable=protected-access + with manager.lock(): tx_id = manager.apply_dtbo(manager.overlay_gen.generate_pool_overlay(diff)) click.echo(f"✓ Pool updated (transaction {tx_id})") diff --git a/src/kerf/runtime.py b/src/kerf/runtime.py index 078b6ac..a4d7820 100644 --- a/src/kerf/runtime.py +++ b/src/kerf/runtime.py @@ -209,8 +209,10 @@ def apply_dtbo(self, dtbo_data: bytes) -> str: if status not in ("applied", "success", "ok"): error_msg = f"Overlay transaction {tx_id} failed with status: '{status}'" try: - instance_name = (tx_dir / "instance").read_text(encoding="utf-8").strip() - error_msg += f" (instance: {instance_name})" + # The file is named 'instance' but holds the target path of + # the overlay's first fragment. + target = (tx_dir / "instance").read_text(encoding="utf-8").strip() + error_msg += f" (target: {target})" except OSError: pass raise KernelInterfaceError(error_msg) @@ -279,7 +281,7 @@ def apply_removal_overlay(self, instance_name: str) -> str: Raises: KernelInterfaceError: If overlay application fails """ - with self._acquire_lock(): + with self.lock(): try: dtbo_data = self.overlay_gen.generate_removal_overlay(instance_name) except Exception as e: @@ -377,7 +379,7 @@ def list_transactions(self) -> List[Dict[str, str]]: return transactions @contextmanager - def _acquire_lock(self): + def lock(self): """ Acquire file lock for concurrency safety. @@ -446,7 +448,7 @@ def apply_operation(self, operation: Callable[[GlobalDeviceTree], GlobalDeviceTr KernelInterfaceError: If kernel interface operations fail Any exceptions raised by the operation function """ - with self._acquire_lock(): + with self.lock(): current = self.read_baseline() # Apply operation (returns modified state) diff --git a/src/kerf/show/main.py b/src/kerf/show/main.py index 5119517..f260c15 100644 --- a/src/kerf/show/main.py +++ b/src/kerf/show/main.py @@ -33,6 +33,7 @@ from ..exceptions import KernelInterfaceError, ParseError from ..metadata import load_instance_metadata from ..models import GlobalDeviceTree +from ..pool_diff import ANY_NODE from ..resources import get_pool_allocated_bytes from ..utils import get_instance_id_from_name, get_instance_status @@ -317,18 +318,24 @@ def display_baseline_info(tree: GlobalDeviceTree, verbose: bool = False): # instance allocations carved out of them; the baseline tree only # snapshots the pool as of the last transaction. click.echo("\n Memory Pool:") - for region in hardware.memory.regions: - node = f" node {region.node}" if region.node >= 0 else "" - click.echo(f" Chunk: {hex(region.base)} {region.size / (1024**3):.2f} GB{node}") - - usage = get_pool_allocated_bytes() - if usage is not None: - _, pool_bytes, allocated_bytes = usage - available_bytes = pool_bytes - allocated_bytes - allocated_gb = allocated_bytes / (1024**3) - available_gb = available_bytes / (1024**3) - click.echo(f" Pool Allocated: {allocated_gb:.2f} GB ({allocated_bytes} bytes)") - click.echo(f" Pool Available: {available_gb:.2f} GB ({available_bytes} bytes)") + if not hardware.memory.regions: + click.echo(" No memory pool configured") + else: + for region in hardware.memory.regions: + node = "" if region.node == ANY_NODE else f" node {region.node}" + click.echo( + f" Chunk: {hex(region.base)} " + f"{region.size / (1024**3):.2f} GB{node}" + ) + + usage = get_pool_allocated_bytes() + if usage is not None: + _, pool_bytes, allocated_bytes = usage + available_bytes = pool_bytes - allocated_bytes + allocated_gb = allocated_bytes / (1024**3) + available_gb = available_bytes / (1024**3) + click.echo(f" Pool Allocated: {allocated_gb:.2f} GB ({allocated_bytes} bytes)") + click.echo(f" Pool Available: {available_gb:.2f} GB ({available_bytes} bytes)") # NUMA Topology if hardware.topology and hardware.topology.numa_nodes: @@ -563,10 +570,15 @@ def show(name: Optional[str], verbose: bool): baseline_manager = BaselineManager() try: tree = baseline_manager.read_baseline() - display_baseline_info(tree, verbose) - except (KernelInterfaceError, ParseError) as e: + except ParseError: + # Once the pool is torn down /resources describes the host and + # carries no pool branch to parse. + click.echo("\nNo memory pool configured") + except KernelInterfaceError as e: if verbose: click.echo(f"\nWarning: Could not read baseline: {e}", err=True) + else: + display_baseline_info(tree, verbose) if not instance_names: click.echo("\n" + "=" * 80) diff --git a/src/kerf/update/main.py b/src/kerf/update/main.py index c65e94b..682a44c 100644 --- a/src/kerf/update/main.py +++ b/src/kerf/update/main.py @@ -352,7 +352,7 @@ def apply_update_operation(current): dtbo_data = manager.overlay_gen.generate_update_overlay(name, old_instance, new_instance) return dtbo_data - with manager._acquire_lock(): # pylint: disable=protected-access + with manager.lock(): current = manager.read_baseline() dtbo_data = apply_update_operation(current) diff --git a/tests/test_init_reconcile.py b/tests/test_init_reconcile.py index 50a29db..1b7f088 100644 --- a/tests/test_init_reconcile.py +++ b/tests/test_init_reconcile.py @@ -42,7 +42,7 @@ def __init__(self): self.applied = [] @contextlib.contextmanager - def _acquire_lock(self): + def lock(self): yield def apply_dtbo(self, dtbo_data): diff --git a/tests/test_runtime.py b/tests/test_runtime.py index e759dc0..e896e9a 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -184,7 +184,7 @@ def test_lock_acquisition(self): assert not manager.lock_file.exists() # Acquire lock - with manager._acquire_lock(): # pylint: disable=protected-access + with manager.lock(): # Lock should exist assert manager.lock_file.exists() @@ -203,7 +203,7 @@ def test_lock_timeout(self): try: # Try to acquire lock (should timeout) with pytest.raises(KernelInterfaceError, match="Could not acquire lock"): - with manager._acquire_lock(): # pylint: disable=protected-access + with manager.lock(): pass finally: # Cleanup diff --git a/tests/test_show.py b/tests/test_show.py index 8e31499..021709a 100644 --- a/tests/test_show.py +++ b/tests/test_show.py @@ -136,3 +136,14 @@ def test_cpu_available_line_kept_when_free_subset_unknown(capsys, monkeypatch): assert "Available: 28 cpus" in out assert "Pool CPUs" not in out assert "Available CPUs" not in out + + +def test_pool_without_chunks_says_so(capsys, monkeypatch): + memory = MemoryAllocation(total_bytes=0, host_reserved_bytes=0, regions=[]) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(_cpus(), memory)) + out = capsys.readouterr().out + + assert "No memory pool configured" in out + assert "Chunk:" not in out From de630203f805a4281dc8d47e98d1373f66bc4fde Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Fri, 21 Aug 2026 13:30:40 -0700 Subject: [PATCH 18/23] pool: Never offer a busy chunk back to the host The diff preferred idle chunks but still fell back to a chunk holding a running instance when no idle chunk was small enough to cover the surplus; the kernel refuses that chunk and fails the whole transaction, so kerf init turned an unshrinkable pool into an error instead of the note it already prints for that case. Only whole idle chunks go back, so never propose a busy one. Signed-off-by: Cong Wang --- src/kerf/pool_diff.py | 6 ++++-- tests/test_pool_diff.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/kerf/pool_diff.py b/src/kerf/pool_diff.py index ce4d814..abab1cd 100644 --- a/src/kerf/pool_diff.py +++ b/src/kerf/pool_diff.py @@ -66,8 +66,10 @@ def _memory_diff(regions: List[PoolMemoryRegion], requested: Dict[int, int], def _release(remaining: List[PoolMemoryRegion], pred, surplus: int, busy: Set[int], diff: PoolDiff) -> None: - candidates = sorted((r for r in remaining if pred(r)), - key=lambda r: (r.base in busy, -r.size)) + # A chunk that still holds an allocation cannot go back to the host, and + # asking anyway fails the whole transaction. + candidates = sorted((r for r in remaining if pred(r) and r.base not in busy), + key=lambda r: -r.size) for r in candidates: if surplus <= 0: break diff --git a/tests/test_pool_diff.py b/tests/test_pool_diff.py index 263b00c..0360e5f 100644 --- a/tests/test_pool_diff.py +++ b/tests/test_pool_diff.py @@ -92,6 +92,17 @@ def test_shrink_prefers_larger_idle_chunks(): assert d.memory_to_pool == [] +def test_busy_chunks_are_never_offered_to_the_host(): + # The only chunk that would fit the surplus holds a running instance. + idle = PoolMemoryRegion(0x1_0000_0000, GB, 0) + busy = PoolMemoryRegion(0x2_0000_0000, GB // 2, 0) + cur = _tree([4], regions=[idle, busy]) + req = _tree([4], requested={0: GB}) + d = compute_pool_diff(cur, req, busy_chunks={busy.base}) + assert d.memory_to_host == [] + assert d.is_empty() + + def test_surplus_smaller_than_every_chunk_stays_in_the_pool(): # Only whole chunks go back to the host, so a 1GB pool asked to shrink to # 512MB keeps its chunk and the caller has to be told. From 65b61e40a20e8bab0be204564d82ba8434efea03 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sat, 22 Aug 2026 09:24:20 -0700 Subject: [PATCH 19/23] init: Request per-node pool memory as SIZE@N Switch the per-node syntax of --memory from nodeK:SIZE to SIZE@N, matching the device-tree unit address convention and the kernel's own memory@N read-back instead of inventing a bespoke node prefix; a plain SIZE still means an unpinned request and the two forms cannot be mixed. Signed-off-by: Cong Wang --- README.md | 2 +- src/kerf/init/main.py | 30 +++++++++++++++++++----------- tests/test_init_memory_spec.py | 9 ++++++--- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 9dc41d1..54285a6 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ Baseline DTB (static) kerf init --cpus=4-7 --memory=2GB # Initialize with CPUs, per-node memory and devices -kerf init --cpus=4-31 --memory=node0:8GB,node1:8GB --devices=enp9s0_dev,nvme0 +kerf init --cpus=4-31 --memory=8GB@0,8GB@1 --devices=enp9s0_dev,nvme0 # Re-run to reshape the live pool, or hand everything back to the host kerf init --cpus=4-15 --memory=4GB diff --git a/src/kerf/init/main.py b/src/kerf/init/main.py index de562bc..7ee0451 100644 --- a/src/kerf/init/main.py +++ b/src/kerf/init/main.py @@ -354,7 +354,7 @@ def get_valid_apic_ids_from_system() -> Optional[set]: return None -_NODE_SPEC = re.compile(r"^node(\d+):(.+)$") +_NODE_SPEC = re.compile(r"^(.+)@(.*)$") PAGE_SIZE = 4096 @@ -383,8 +383,9 @@ def parse_memory_request(spec: str) -> Dict[int, int]: """ Parse a pool memory request into per-NUMA-node sizes. - "2GB" asks for 2GB on any node (node -1), "node0:8GB,node1:8GB" asks - for a specific amount per node. The two forms cannot be mixed. + "2GB" asks for 2GB on any node (node -1), "8GB@0,8GB@1" asks for a + specific amount per node, mirroring the device-tree unit address + convention. The two forms cannot be mixed. Args: spec: Memory specification string @@ -404,9 +405,16 @@ def parse_memory_request(spec: str) -> Dict[int, int]: for part in parts: match = _NODE_SPEC.match(part) if match: - node, size = int(match.group(1)), parse_memory_spec(match.group(2)) - elif part.lower().startswith("node"): - raise ValueError(f"invalid NUMA node spec '{part}' (expected nodeN:SIZE)") + size_part, node_part = match.group(1), match.group(2) + try: + node = int(node_part) + except ValueError as exc: + raise ValueError( + f"invalid NUMA node '{node_part}' in '{part}' (expected SIZE@N)" + ) from exc + if node < 0: + raise ValueError(f"NUMA node in '{part}' must not be negative") + size = parse_memory_spec(size_part) else: node, size = ANY_NODE, parse_memory_spec(part) if node in requested: @@ -414,7 +422,7 @@ def parse_memory_request(spec: str) -> Dict[int, int]: requested[node] = size if ANY_NODE in requested and len(requested) > 1: - raise ValueError("cannot mix a plain size with nodeN: sizes") + raise ValueError("cannot mix a plain size with SIZE@N entries") validate_memory_request(requested) return requested @@ -432,7 +440,7 @@ def build_baseline_from_cmdline( Args: cpus: CPU specification string (e.g., "4-7" or "4,5,6,7") memory: Pool memory request, either "2GB" for any node or - "node0:8GB,node1:8GB" for specific nodes + "8GB@0,8GB@1" for specific nodes devices: Optional device names (comma-separated, e.g., "enp9s0_dev,nvme0") verbose: Whether to print verbose output pool_cpus: APIC IDs the pool already holds @@ -765,7 +773,7 @@ def _dump_baseline_dts(baseline_mgr: BaselineManager, tree: GlobalDeviceTree) -> @click.pass_context @click.option('--input', '-i', help='Input DTS or DTB file containing all resources. Mutually exclusive with --cpus, --memory and --devices. When used, all resources must come from the file.') @click.option('--cpus', '-c', help='APIC ID specification for baseline (e.g., "128-134" or "128,130,132"). Use physical APIC IDs, not logical CPU numbers. Mutually exclusive with --input.') -@click.option('--memory', '-m', help='Pool memory: SIZE for any node (e.g. "2GB") or per-node "node0:8GB,node1:8GB". Required with --cpus, mutually exclusive with --input.') +@click.option('--memory', '-m', help='Pool memory: SIZE for any node (e.g. "2GB") or per-node "8GB@0,8GB@1". Required with --cpus, mutually exclusive with --input.') @click.option('--devices', '-d', help='Device names (comma-separated, e.g., "enp9s0_dev,nvme0"). Mutually exclusive with --input. Creates minimal device entries in baseline.') @click.option('--teardown', is_flag=True, help='Return every pool resource to the host. Mutually exclusive with --input, --cpus, --memory and --devices.') @click.option('--dry-run', is_flag=True, help='Report the plan without applying it. Still reads the pool from the kernel, so it needs root.') @@ -800,7 +808,7 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: kerf init --cpus=128-134 --memory=1GB # Request memory per NUMA node - kerf init --cpus=128-134 --memory=node0:8GB,node1:8GB + kerf init --cpus=128-134 --memory=8GB@0,8GB@1 # Shrink the pool back to 1GB and 2 CPUs kerf init --cpus=128,129 --memory=1GB @@ -837,7 +845,7 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: click.echo("\nUsage:", err=True) click.echo(" kerf init --input=hardware.dts", err=True) click.echo(" kerf init --cpus=4-7 --memory=1GB", err=True) - click.echo(" kerf init --cpus=4-7 --memory=node0:1GB --devices=enp9s0_dev", err=True) + click.echo(" kerf init --cpus=4-7 --memory=1GB@0 --devices=enp9s0_dev", err=True) click.echo(" kerf init --teardown", err=True) sys.exit(2) diff --git a/tests/test_init_memory_spec.py b/tests/test_init_memory_spec.py index e90afb1..565011b 100644 --- a/tests/test_init_memory_spec.py +++ b/tests/test_init_memory_spec.py @@ -26,16 +26,19 @@ def test_plain_size_is_any_node(): def test_per_node_sizes(): - assert parse_memory_request("node0:8GB, node1:512MB") == {0: 8 * GB, 1: 512 << 20} + assert parse_memory_request("8GB@0, 512MB@1") == {0: 8 * GB, 1: 512 << 20} -@pytest.mark.parametrize("spec", ["node0:1GB,2GB", "node0:1GB,node0:1GB", "nodeX:1GB", ""]) +@pytest.mark.parametrize( + "spec", + ["8GB@x", "8GB@-1", "8GB@0,8GB", "8GB@0,4GB@0", "@0", "8GB@", ""], +) def test_invalid_specs(spec): with pytest.raises(ValueError): parse_memory_request(spec) -@pytest.mark.parametrize("spec", ["0", "node0:0", "4097", "node1:5000"]) +@pytest.mark.parametrize("spec", ["0", "0@0", "4097", "5000@1"]) def test_sizes_must_be_positive_and_page_aligned(spec): with pytest.raises(ValueError): parse_memory_request(spec) From cfb764ed7d81097e3a329c127587d4167c89937c Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sat, 22 Aug 2026 09:39:04 -0700 Subject: [PATCH 20/23] init: Pick the NUMA node for an unpinned memory request in user space A --memory size without an @N used to reach the kernel with no numa- node-id, leaving the kernel to choose where the pool memory came from; placement is policy and policy belongs in kerf. Resolve the node here, from the NUMA node of the requested CPUs (the node they share, or the lowest APIC id's node when they straddle nodes), falling back to the node of the chunks the pool already holds when its CPUs are offline and the host no longer places them, and to node 0 when the host reports no topology at all; baselines read from --input get the same treatment, so the kernel is always handed an explicit node. Signed-off-by: Cong Wang --- README.md | 1 + src/kerf/init/main.py | 124 +++++++++++++++++--- src/kerf/topology.py | 139 +++++++++++++++++++++++ tests/test_init_numa_node.py | 212 +++++++++++++++++++++++++++++++++++ tests/test_topology.py | 106 ++++++++++++++++++ 5 files changed, 569 insertions(+), 13 deletions(-) create mode 100644 src/kerf/topology.py create mode 100644 tests/test_init_numa_node.py create mode 100644 tests/test_topology.py diff --git a/README.md b/README.md index 54285a6..7f7a666 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,7 @@ Baseline DTB (static) ### Command Line Interface ```bash # Initialize resource pool with CPUs and pool memory +# A plain size lands on the NUMA node of the requested CPUs kerf init --cpus=4-7 --memory=2GB # Initialize with CPUs, per-node memory and devices diff --git a/src/kerf/init/main.py b/src/kerf/init/main.py index 7ee0451..8c95bde 100644 --- a/src/kerf/init/main.py +++ b/src/kerf/init/main.py @@ -25,7 +25,7 @@ import re import sys from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple import click import libfdt @@ -47,10 +47,12 @@ GlobalDeviceTree, HardwareInventory, MemoryAllocation, + PoolMemoryRegion, ) from ..pool_diff import ANY_NODE, PoolDiff, compute_pool_diff from ..resources import get_busy_chunks_from_iomem from ..runtime import DeviceTreeManager +from ..topology import cpu_numa_nodes, node_for_cpus MULTIKERNEL_MOUNT_POINT = "/sys/fs/multikernel" @@ -364,13 +366,13 @@ def validate_memory_request(requested: Dict[int, int]) -> None: Reject pool sizes the kernel cannot honour, however they were asked for. Args: - requested: Mapping of NUMA node id (ANY_NODE for any node) to size + requested: Mapping of NUMA node id (ANY_NODE if not resolved yet) to size Raises: ValueError: If a size is zero, negative or not page aligned """ for node, size in sorted(requested.items()): - where = "any node" if node == ANY_NODE else f"node {node}" + where = "the unpinned request" if node == ANY_NODE else f"node {node}" if size <= 0: raise ValueError(f"memory size for {where} must be greater than zero") if size % PAGE_SIZE: @@ -383,7 +385,8 @@ def parse_memory_request(spec: str) -> Dict[int, int]: """ Parse a pool memory request into per-NUMA-node sizes. - "2GB" asks for 2GB on any node (node -1), "8GB@0,8GB@1" asks for a + "2GB" asks for 2GB without naming a node, which the caller then + resolves to the node of the requested CPUs; "8GB@0,8GB@1" asks for a specific amount per node, mirroring the device-tree unit address convention. The two forms cannot be mixed. @@ -391,7 +394,7 @@ def parse_memory_request(spec: str) -> Dict[int, int]: spec: Memory specification string Returns: - Mapping of NUMA node id (-1 for any node) to size in bytes + Mapping of NUMA node id (-1 when the node is left to kerf) to size in bytes Raises: ValueError: If the specification is malformed or a size is zero @@ -427,23 +430,96 @@ def parse_memory_request(spec: str) -> Dict[int, int]: return requested +def _cpu_ranges(cpus: List[int]) -> str: + """Render APIC ids the way they are written on the command line.""" + parts = [] + for cpu in sorted(set(cpus)): + if parts and cpu == parts[-1][1] + 1: + parts[-1][1] = cpu + else: + parts.append([cpu, cpu]) + return ",".join(str(a) if a == b else f"{a}-{b}" for a, b in parts) + + +def pick_memory_node(cpu_list: List[int], + pool_cpus: Optional[set] = None, + pool_regions: Optional[List[PoolMemoryRegion]] = None + ) -> Tuple[int, str]: + """ + Choose the NUMA node for a memory request that did not name one. + + Placement is policy, so kerf decides it and the kernel is only ever + handed an explicit node. + + Args: + cpu_list: APIC IDs the pool is being asked for + pool_cpus: APIC IDs the pool already holds + pool_regions: Chunks the pool already holds + + Returns: + The chosen node and a short reason to show the user + """ + node = node_for_cpus(cpu_list, cpu_numa_nodes()) + if node is not None: + return node, f"from CPUs {_cpu_ranges(cpu_list)}" + + # A CPU the pool already took is offline, so neither sysfs nor + # /proc/cpuinfo places it any more; its chunks still do. + if set(cpu_list) & set(pool_cpus or ()): + nodes = {r.node for r in (pool_regions or []) if r.node != ANY_NODE} + if len(nodes) == 1: + return nodes.pop(), "from the chunks the pool already holds" + + return 0, "no NUMA topology available, defaulting to node 0" + + +def resolve_memory_nodes(requested: Dict[int, int], + cpu_list: List[int], + pool_cpus: Optional[set] = None, + pool_regions: Optional[List[PoolMemoryRegion]] = None + ) -> Tuple[Dict[int, int], Optional[str]]: + """ + Pin an unpinned memory request to a node before the kernel sees it. + + Args: + requested: Mapping of NUMA node id (ANY_NODE for unpinned) to size + cpu_list: APIC IDs the pool is being asked for + pool_cpus: APIC IDs the pool already holds + pool_regions: Chunks the pool already holds + + Returns: + The request with every size on an explicit node, and a line + describing the choice, or None if nothing had to be resolved + """ + resolved = dict(requested) + size = resolved.pop(ANY_NODE, None) + if size is None: + return resolved, None + + node, why = pick_memory_node(cpu_list, pool_cpus, pool_regions) + resolved[node] = resolved.get(node, 0) + size + return resolved, f"Memory: {size >> 20} MB on node {node} ({why})" + + def build_baseline_from_cmdline( cpus: str, memory: Optional[str] = None, devices: Optional[str] = None, verbose: bool = False, - pool_cpus: Optional[set] = None + pool_cpus: Optional[set] = None, + pool_regions: Optional[List[PoolMemoryRegion]] = None ) -> GlobalDeviceTree: """ Build a GlobalDeviceTree from command line arguments. Args: cpus: CPU specification string (e.g., "4-7" or "4,5,6,7") - memory: Pool memory request, either "2GB" for any node or - "8GB@0,8GB@1" for specific nodes + memory: Pool memory request, either "2GB" on the node of the + requested CPUs or "8GB@0,8GB@1" for specific nodes devices: Optional device names (comma-separated, e.g., "enp9s0_dev,nvme0") verbose: Whether to print verbose output pool_cpus: APIC IDs the pool already holds + pool_regions: Chunks the pool already holds Returns: GlobalDeviceTree with resources only (no instances) @@ -493,6 +569,10 @@ def build_baseline_from_cmdline( host_reserved_cpus = [0] cpu_list = sorted(list(available_cpus)) + requested, note = resolve_memory_nodes(requested, cpu_list, pool_cpus, pool_regions) + if note: + click.echo(note) + total_bytes = sum(requested.values()) if verbose: click.echo(f"Parsed APIC ID specification: {cpus}") @@ -501,8 +581,7 @@ def build_baseline_from_cmdline( click.echo(f" Available APIC IDs: {cpu_list}") click.echo("Requested pool memory:") for node, size in sorted(requested.items()): - where = "any node" if node < 0 else f"node {node}" - click.echo(f" {where}: {size} bytes ({size / (1024**3):.2f} GB)") + click.echo(f" node {node}: {size} bytes ({size / (1024**3):.2f} GB)") cpu_allocation = CPUAllocation( total=total_cpus, @@ -611,6 +690,13 @@ def pool_apic_ids(current: Optional[GlobalDeviceTree]) -> set: return set(current.hardware.cpus.available or []) +def pool_memory_regions(current: Optional[GlobalDeviceTree]) -> List[PoolMemoryRegion]: + """Chunks the live pool holds, which place CPUs the host no longer reports.""" + if not pool_is_live(current): + return [] + return list(current.hardware.memory.regions) + + def read_current_pool(baseline_mgr) -> Optional[GlobalDeviceTree]: """ Read the live pool back from the kernel. @@ -773,7 +859,7 @@ def _dump_baseline_dts(baseline_mgr: BaselineManager, tree: GlobalDeviceTree) -> @click.pass_context @click.option('--input', '-i', help='Input DTS or DTB file containing all resources. Mutually exclusive with --cpus, --memory and --devices. When used, all resources must come from the file.') @click.option('--cpus', '-c', help='APIC ID specification for baseline (e.g., "128-134" or "128,130,132"). Use physical APIC IDs, not logical CPU numbers. Mutually exclusive with --input.') -@click.option('--memory', '-m', help='Pool memory: SIZE for any node (e.g. "2GB") or per-node "8GB@0,8GB@1". Required with --cpus, mutually exclusive with --input.') +@click.option('--memory', '-m', help='Pool memory: SIZE (e.g. "2GB") on the node of the requested CPUs, or per-node "8GB@0,8GB@1". Required with --cpus, mutually exclusive with --input.') @click.option('--devices', '-d', help='Device names (comma-separated, e.g., "enp9s0_dev,nvme0"). Mutually exclusive with --input. Creates minimal device entries in baseline.') @click.option('--teardown', is_flag=True, help='Return every pool resource to the host. Mutually exclusive with --input, --cpus, --memory and --devices.') @click.option('--dry-run', is_flag=True, help='Report the plan without applying it. Still reads the pool from the kernel, so it needs root.') @@ -800,11 +886,15 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: request from command line arguments using --cpus and --memory. These options are mutually exclusive. + A --memory size that names no node is placed on the NUMA node of the + requested CPUs. Kerf resolves it here so the kernel is always handed an + explicit node and never picks the placement itself. + Examples: # Initialize from DTS file (all resources from file) kerf init --input=hardware.dts - # Request 1GB of pool memory on any node + # Request 1GB of pool memory on the node of the requested CPUs kerf init --cpus=128-134 --memory=1GB # Request memory per NUMA node @@ -857,6 +947,7 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: mount_multikernel_fs(verbose=verbose) current = read_current_pool(baseline_mgr) live_cpus = pool_apic_ids(current) + live_regions = pool_memory_regions(current) if teardown: tree = build_teardown_tree(pool_cpus=live_cpus) @@ -884,11 +975,18 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: except ValueError as e: click.echo(f"Error: {input}: {e}", err=True) sys.exit(2) + + tree.hardware.memory.requested, note = resolve_memory_nodes( + tree.hardware.memory.requested, tree.hardware.cpus.available, + live_cpus, live_regions) + if note: + click.echo(note) else: # Build from command line arguments try: tree = build_baseline_from_cmdline(cpus, memory=memory, devices=devices, - verbose=verbose, pool_cpus=live_cpus) + verbose=verbose, pool_cpus=live_cpus, + pool_regions=live_regions) except ValueError as e: click.echo(f"Error: {e}", err=True) sys.exit(2) diff --git a/src/kerf/topology.py b/src/kerf/topology.py new file mode 100644 index 0000000..1537b16 --- /dev/null +++ b/src/kerf/topology.py @@ -0,0 +1,139 @@ +# Copyright 2026 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Host NUMA topology, as sysfs and /proc/cpuinfo describe it.""" + +import re +from pathlib import Path +from typing import Dict, Iterable, List, Optional + +NODE_ROOT = "/sys/devices/system/node" +CPUINFO = "/proc/cpuinfo" + +_NODE_DIR = re.compile(r"^node(\d+)$") + + +def _parse_cpulist(text: str) -> List[int]: + """Expand a sysfs cpulist ("0-3,8") into logical CPU numbers.""" + cpus: List[int] = [] + for part in text.strip().split(","): + part = part.strip() + if not part: + continue + try: + if "-" in part: + start, _, end = part.partition("-") + cpus.extend(range(int(start), int(end) + 1)) + else: + cpus.append(int(part)) + except ValueError: + continue + return cpus + + +def logical_to_apic(cpuinfo_path: str = CPUINFO) -> Dict[int, int]: + """ + Map logical CPU number to APIC id. + + Args: + cpuinfo_path: Path to read instead of /proc/cpuinfo + + Returns: + Mapping of logical CPU number to APIC id, empty if unreadable + """ + mapping: Dict[int, int] = {} + processor: Optional[int] = None + try: + with open(cpuinfo_path, "r", encoding="utf-8") as f: + for line in f: + key, sep, value = line.partition(":") + if not sep: + continue + key, value = key.strip(), value.strip() + if key == "processor": + processor = _int_or_none(value) + elif key == "apicid" and processor is not None: + apic = _int_or_none(value) + if apic is not None: + mapping[processor] = apic + processor = None + except OSError: + return {} + return mapping + + +def _int_or_none(value: str) -> Optional[int]: + try: + return int(value) + except ValueError: + return None + + +def cpu_numa_nodes(node_root: str = NODE_ROOT, cpuinfo_path: str = CPUINFO) -> Dict[int, int]: + """ + Map APIC id to NUMA node for every CPU the host still reports. + + Args: + node_root: Path to read instead of /sys/devices/system/node + cpuinfo_path: Path to read instead of /proc/cpuinfo + + Returns: + Mapping of APIC id to NUMA node, empty if the topology is unreadable + """ + apic_of = logical_to_apic(cpuinfo_path) + if not apic_of: + return {} + + try: + entries = sorted(Path(node_root).iterdir()) + except OSError: + return {} + + nodes: Dict[int, int] = {} + for entry in entries: + match = _NODE_DIR.match(entry.name) + if not match: + continue + try: + cpulist = (entry / "cpulist").read_text(encoding="utf-8") + except OSError: + continue + for cpu in _parse_cpulist(cpulist): + apic = apic_of.get(cpu) + if apic is not None: + nodes[apic] = int(match.group(1)) + return nodes + + +def node_for_cpus(apic_ids: Iterable[int], mapping: Dict[int, int]) -> Optional[int]: + """ + Pick the NUMA node these CPUs belong to. + + A request has to name one node, so CPUs spread over several follow the + lowest APIC id rather than leaving the choice to the kernel. + + Args: + apic_ids: APIC ids of the requested CPUs + mapping: APIC id to NUMA node mapping from cpu_numa_nodes() + + Returns: + The chosen node, or None if no requested CPU has a known node + """ + known = sorted(apic for apic in apic_ids if apic in mapping) + if not known: + return None + nodes = {mapping[apic] for apic in known} + if len(nodes) == 1: + return nodes.pop() + return mapping[known[0]] diff --git a/tests/test_init_numa_node.py b/tests/test_init_numa_node.py new file mode 100644 index 0000000..e8fb024 --- /dev/null +++ b/tests/test_init_numa_node.py @@ -0,0 +1,212 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The NUMA node 'kerf init' picks for a memory request that names none.""" + +import pytest +from click.testing import CliRunner + +from kerf.dtc.overlay import OverlayGenerator +from kerf.dtc.parser import DeviceTreeParser +from kerf.exceptions import ParseError +from kerf.init import main +from kerf.models import ( + CPUAllocation, + GlobalDeviceTree, + HardwareInventory, + MemoryAllocation, + PoolMemoryRegion, +) +from kerf.pool_diff import ANY_NODE, compute_pool_diff + +MB = 1 << 20 +GB = 1 << 30 + + +def _live_pool(cpus, regions): + """What the kernel reports for a pool that already holds these resources.""" + return GlobalDeviceTree( + hardware=HardwareInventory( + cpus=CPUAllocation(total=16, host_reserved=[0], available=cpus), + memory=MemoryAllocation(total_bytes=0, host_reserved_bytes=0, regions=regions), + devices={}, + ), + instances={}, + device_references={}, + ) + + +@pytest.fixture(name="topology") +def topology_fixture(monkeypatch): + """Let a test state the APIC id to node mapping the host reports.""" + def install(mapping): + monkeypatch.setattr(main, "cpu_numa_nodes", lambda: dict(mapping)) + install({}) + return install + + +def test_cpus_on_one_node_pin_the_request_there(topology): + topology({1: 1, 2: 1, 3: 1}) + + requested, note = main.resolve_memory_nodes({ANY_NODE: 512 * MB}, [1, 2, 3]) + + assert requested == {1: 512 * MB} + assert note == "Memory: 512 MB on node 1 (from CPUs 1-3)" + + +def test_cpus_split_across_nodes_follow_the_lowest_apic_id(topology): + topology({2: 1, 5: 0}) + + requested, note = main.resolve_memory_nodes({ANY_NODE: GB}, [5, 2]) + + assert requested == {1: GB} + assert "from CPUs 2,5" in note + + +def test_pool_cpus_fall_back_to_the_chunks_they_run_on(topology): + # A CPU the pool already holds is offline, so the topology cannot place it. + topology({0: 0}) + regions = [PoolMemoryRegion(0x1_0000_0000, GB, 1)] + + requested, note = main.resolve_memory_nodes( + {ANY_NODE: GB}, [1, 2, 3], pool_cpus={1, 2, 3}, pool_regions=regions) + + assert requested == {1: GB} + assert "chunks the pool already holds" in note + + +def test_chunks_on_several_nodes_do_not_decide(topology): + topology({0: 0}) + regions = [PoolMemoryRegion(0x1_0000_0000, GB, 0), + PoolMemoryRegion(0x2_0000_0000, GB, 1)] + + requested, note = main.resolve_memory_nodes( + {ANY_NODE: GB}, [1, 2], pool_cpus={1, 2}, pool_regions=regions) + + assert requested == {0: GB} + assert "no NUMA topology available" in note + + +def test_no_numa_information_defaults_to_node_zero(topology): + topology({}) + + requested, note = main.resolve_memory_nodes({ANY_NODE: 512 * MB}, [1, 2, 3]) + + assert requested == {0: 512 * MB} + assert note == "Memory: 512 MB on node 0 (no NUMA topology available, defaulting to node 0)" + + +def test_explicit_nodes_are_left_alone(topology): + topology({1: 1}) + + requested, note = main.resolve_memory_nodes({0: GB, 1: 2 * GB}, [1]) + + assert requested == {0: GB, 1: 2 * GB} + assert note is None + + +def test_a_resolved_size_joins_an_explicit_request_on_the_same_node(topology): + topology({4: 0}) + + requested, _ = main.resolve_memory_nodes({0: GB, ANY_NODE: GB}, [4]) + + assert requested == {0: 2 * GB} + + +def test_the_request_built_from_the_command_line_names_a_node(topology, monkeypatch): + topology({0: 0, 1: 1, 2: 1, 3: 1}) + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1, 2, 3}) + + tree = main.build_baseline_from_cmdline("1-3", memory="512MB") + + assert tree.hardware.memory.requested == {1: 512 * MB} + + +def test_the_diff_sees_an_explicit_node(topology, monkeypatch): + topology({0: 0, 1: 1, 2: 1, 3: 1}) + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1, 2, 3}) + current = _live_pool([1, 2, 3], [PoolMemoryRegion(0x1_0000_0000, GB, 1)]) + + requested = main.build_baseline_from_cmdline("1-3", memory="2GB", + pool_cpus=main.pool_apic_ids(current)) + diff = compute_pool_diff(current, requested) + + assert diff.memory_to_pool == [(1, GB)] + + +_DTS_UNPINNED = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + + memory@0 { + size = <0x0 0x40000000>; + }; + }; +}; +""" + + +def test_a_baseline_file_without_numa_node_id_is_resolved_too(topology): + topology({4: 1, 5: 1}) + tree = DeviceTreeParser().parse_dts(_DTS_UNPINNED) + assert tree.hardware.memory.requested == {ANY_NODE: GB} + + requested, note = main.resolve_memory_nodes( + tree.hardware.memory.requested, tree.hardware.cpus.available) + + assert requested == {1: GB} + assert "from CPUs 4-5" in note + + +class _FakeBaselineManager: + """Stands in for BaselineManager: no pool yet, records the baseline write.""" + + def __init__(self): + self.written = [] + + def validate_baseline(self, tree): + """Accept anything: the baseline shape is covered elsewhere.""" + + def read_baseline(self): + raise ParseError("No memory description in /resources") + + def write_baseline(self, tree): + self.written.append(tree) + + +class _FakeManager: + """Stands in for DeviceTreeManager: never touches sysfs.""" + + def __init__(self, *_args, **_kwargs): + self.overlay_gen = OverlayGenerator() + + +def test_init_resolves_a_baseline_file_before_writing_it(topology, monkeypatch, tmp_path): + topology({4: 1, 5: 1}) + baseline_mgr = _FakeBaselineManager() + monkeypatch.setattr(main, "mount_multikernel_fs", lambda verbose=False: None) + monkeypatch.setattr(main, "DeviceTreeManager", _FakeManager) + monkeypatch.setattr(main, "get_busy_chunks_from_iomem", set) + monkeypatch.setattr(main, "BaselineManager", lambda *a, **kw: baseline_mgr) + dts = tmp_path / "baseline.dts" + dts.write_text(_DTS_UNPINNED, encoding="utf-8") + + result = CliRunner().invoke(main.init, [f"--input={dts}"]) + + assert result.exit_code == 0, result.output + assert "Memory: 1024 MB on node 1 (from CPUs 4-5)" in result.output + assert baseline_mgr.written[-1].hardware.memory.requested == {1: GB} diff --git a/tests/test_topology.py b/tests/test_topology.py new file mode 100644 index 0000000..a6b5329 --- /dev/null +++ b/tests/test_topology.py @@ -0,0 +1,106 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reading the host NUMA topology out of sysfs and /proc/cpuinfo.""" + +from kerf.topology import cpu_numa_nodes, logical_to_apic, node_for_cpus + + +def _cpuinfo(tmp_path, apic_of_cpu): + path = tmp_path / "cpuinfo" + blocks = [] + for cpu, apic in sorted(apic_of_cpu.items()): + blocks.append( + f"processor\t: {cpu}\n" + "vendor_id\t: GenuineIntel\n" + "cpu family\t: 25\n" + f"apicid\t\t: {apic}\n" + f"initial apicid\t: {apic}\n" + ) + path.write_text("\n".join(blocks), encoding="utf-8") + return str(path) + + +def _node_root(tmp_path, cpulists): + root = tmp_path / "node" + root.mkdir() + for node, cpulist in cpulists.items(): + node_dir = root / f"node{node}" + node_dir.mkdir() + (node_dir / "cpulist").write_text(cpulist + "\n", encoding="utf-8") + return str(root) + + +def test_logical_cpus_map_to_apic_ids(tmp_path): + cpuinfo = _cpuinfo(tmp_path, {0: 0, 1: 2, 2: 4}) + + assert logical_to_apic(cpuinfo) == {0: 0, 1: 2, 2: 4} + + +def test_single_node_places_every_cpu(tmp_path): + cpuinfo = _cpuinfo(tmp_path, {0: 0, 1: 2, 2: 4, 3: 6}) + node_root = _node_root(tmp_path, {0: "0-3"}) + + mapping = cpu_numa_nodes(node_root, cpuinfo) + + assert mapping == {0: 0, 2: 0, 4: 0, 6: 0} + assert node_for_cpus([2, 4], mapping) == 0 + + +def test_two_nodes_keep_their_own_cpus(tmp_path): + cpuinfo = _cpuinfo(tmp_path, {0: 0, 1: 2, 2: 4, 3: 6}) + node_root = _node_root(tmp_path, {0: "0,1", 1: "2-3"}) + + mapping = cpu_numa_nodes(node_root, cpuinfo) + + assert mapping == {0: 0, 2: 0, 4: 1, 6: 1} + assert node_for_cpus([4, 6], mapping) == 1 + + +def test_cpus_split_across_nodes_follow_the_lowest_apic_id(tmp_path): + cpuinfo = _cpuinfo(tmp_path, {0: 0, 1: 2, 2: 4, 3: 6}) + node_root = _node_root(tmp_path, {0: "0,1", 1: "2-3"}) + + mapping = cpu_numa_nodes(node_root, cpuinfo) + + assert node_for_cpus([2, 4], mapping) == 0 + assert node_for_cpus([4, 2], mapping) == 0 + + +def test_offline_cpus_are_absent_from_the_mapping(tmp_path): + # The pool's own CPUs leave /proc/cpuinfo and the node cpulist. + cpuinfo = _cpuinfo(tmp_path, {0: 0}) + node_root = _node_root(tmp_path, {0: "0"}) + + mapping = cpu_numa_nodes(node_root, cpuinfo) + + assert node_for_cpus([2, 4], mapping) is None + + +def test_missing_files_leave_the_node_undecided(tmp_path): + missing = str(tmp_path / "nowhere") + + assert cpu_numa_nodes(missing, missing) == {} + assert node_for_cpus([0, 1], {}) is None + + cpuinfo = _cpuinfo(tmp_path, {0: 0}) + assert cpu_numa_nodes(missing, cpuinfo) == {} + assert cpu_numa_nodes(_node_root(tmp_path, {0: "0"}), missing) == {} + + +def test_memoryless_node_with_no_cpus_is_skipped(tmp_path): + cpuinfo = _cpuinfo(tmp_path, {0: 0, 1: 2}) + node_root = _node_root(tmp_path, {0: "0-1", 1: ""}) + + assert cpu_numa_nodes(node_root, cpuinfo) == {0: 0, 2: 0} From bc51ac647066a4583236e062a7b2c40cc4a25552 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sat, 22 Aug 2026 14:58:39 -0700 Subject: [PATCH 21/23] tests: Satisfy the newer pylint run by CI CI runs a pylint release that flags yielding elements one by one where yield from applies and comparing a result against an empty dict instead of testing its truthiness. Neither changes what the tests verify, so adopt the suggested spellings to keep the lint gate green. Signed-off-by: Cong Wang --- tests/test_pool_overlay.py | 3 +-- tests/test_topology.py | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_pool_overlay.py b/tests/test_pool_overlay.py index 7564c97..d0ceb2b 100644 --- a/tests/test_pool_overlay.py +++ b/tests/test_pool_overlay.py @@ -36,8 +36,7 @@ def _walk(fdt, offset=0): yield offset child = fdt.first_subnode(offset, quiet=[libfdt.FDT_ERR_NOTFOUND]) while child >= 0: - for descendant in _walk(fdt, child): - yield descendant + yield from _walk(fdt, child) child = fdt.next_subnode(child, quiet=[libfdt.FDT_ERR_NOTFOUND]) diff --git a/tests/test_topology.py b/tests/test_topology.py index a6b5329..413d8fd 100644 --- a/tests/test_topology.py +++ b/tests/test_topology.py @@ -91,12 +91,12 @@ def test_offline_cpus_are_absent_from_the_mapping(tmp_path): def test_missing_files_leave_the_node_undecided(tmp_path): missing = str(tmp_path / "nowhere") - assert cpu_numa_nodes(missing, missing) == {} + assert not cpu_numa_nodes(missing, missing) assert node_for_cpus([0, 1], {}) is None cpuinfo = _cpuinfo(tmp_path, {0: 0}) - assert cpu_numa_nodes(missing, cpuinfo) == {} - assert cpu_numa_nodes(_node_root(tmp_path, {0: "0"}), missing) == {} + assert not cpu_numa_nodes(missing, cpuinfo) + assert not cpu_numa_nodes(_node_root(tmp_path, {0: "0"}), missing) def test_memoryless_node_with_no_cpus_is_skipped(tmp_path): From 51b48944f34859c4bd939d040b5b693dbb972925 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sat, 22 Aug 2026 15:05:30 -0700 Subject: [PATCH 22/23] ci: Build pylibfdt with a pinned swig before poetry installs Poetry builds sdists through "pip --isolated", which ignores PIP_CONSTRAINT along with every other pip setting, so the previous pin never reached the environment that compiles pylibfdt and its wrapper still came out with the PyInt_AsLong call swig 4.5 no longer maps. Install swig below 4.5 into the project venv, build pylibfdt there without build isolation so it uses that swig, and let poetry install find the package already satisfied. Signed-off-by: Cong Wang --- .github/workflows/lint_and_test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/lint_and_test.yml b/.github/workflows/lint_and_test.yml index 6967808..0beff83 100644 --- a/.github/workflows/lint_and_test.yml +++ b/.github/workflows/lint_and_test.yml @@ -37,6 +37,12 @@ jobs: - name: Install dependencies run: | + # pylibfdt builds its SWIG wrapper from source, and swig 4.5 dropped + # the PyInt_* macros that wrapper still uses. Poetry builds sdists + # with "pip --isolated", which ignores PIP_CONSTRAINT, so build + # pylibfdt ourselves with a pinned swig and let poetry find it. + poetry run python -m pip install "swig<4.5" setuptools setuptools_scm + poetry run python -m pip install --no-build-isolation "pylibfdt>=1.7.0,<2" poetry install --no-interaction - name: Analysing the code with pylint From 8b0a67a5724f0048b386d54f2ed4a78ce8324ce3 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Sat, 22 Aug 2026 15:35:17 -0700 Subject: [PATCH 23/23] init: Spell an empty resource as none and drop --teardown A kerf init request is the desired state of the pool, so an option that names no resource is already the way to say "none of this": --cpus=none, --memory=none (or --memory=0) and --devices=none each ask for none of that resource, and a request that asks for nothing at all returns the whole pool to the host. The --teardown flag only duplicated that one request behind a second spelling, along with a build_teardown_tree() that built the same tree the ordinary path builds once the available lists are empty, so both are gone and a full teardown is now written "kerf init --cpus=none --memory=none". Mixing none with real entries is rejected, --cpus and --memory stay required so a forgotten --memory cannot silently release the pool, and the instance guard, the empty-pool message, the shortfall note and the dry-run plan all keep working off the same "is the request empty" test as before. While here, drop the meaningless "Host Reserved: 0 cpus: []" line kerf show printed for a tree read back from the kernel, which describes the pool rather than the host. Signed-off-by: Cong Wang --- README.md | 7 +- src/kerf/init/main.py | 162 ++++++++++++++++++++++----------- src/kerf/show/main.py | 9 +- tests/test_init_memory_spec.py | 52 ++++++++++- tests/test_init_reconcile.py | 48 +++++++--- tests/test_show.py | 15 ++- 6 files changed, 214 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index 7f7a666..3ff27a6 100644 --- a/README.md +++ b/README.md @@ -134,9 +134,12 @@ kerf init --cpus=4-7 --memory=2GB # Initialize with CPUs, per-node memory and devices kerf init --cpus=4-31 --memory=8GB@0,8GB@1 --devices=enp9s0_dev,nvme0 -# Re-run to reshape the live pool, or hand everything back to the host +# Re-run to reshape the live pool; "none" asks for none of a resource kerf init --cpus=4-15 --memory=4GB -kerf init --teardown +kerf init --cpus=4-15 --memory=none + +# Hand everything back to the host +kerf init --cpus=none --memory=none # Create kernel instance with resource allocation kerf create web-server --cpus=4-7 --memory=2GB diff --git a/src/kerf/init/main.py b/src/kerf/init/main.py index 8c95bde..182c9d6 100644 --- a/src/kerf/init/main.py +++ b/src/kerf/init/main.py @@ -361,6 +361,73 @@ def get_valid_apic_ids_from_system() -> Optional[set]: PAGE_SIZE = 4096 +NO_RESOURCE = "none" + + +def spec_is_empty(spec: str, option: str, synonyms: Tuple[str, ...] = ()) -> bool: + """ + Whether an option spells "none of this resource". + + Args: + spec: The option's value as typed on the command line + option: The option's name, for the error message + synonyms: Extra spellings that mean the same as "none" + + Returns: + True if the option asks for nothing + + Raises: + ValueError: If an empty spelling is mixed with real entries + """ + empty = {NO_RESOURCE} | set(synonyms) + parts = [p.strip().lower() for p in spec.split(",") if p.strip()] + asked = [p for p in parts if p in empty] + if not asked: + return False + if len(parts) > 1: + raise ValueError(f"{option}={asked[0]} cannot be combined with other entries") + return True + + +def parse_cpu_request(spec: str) -> List[int]: + """ + Parse the pool CPU request, where "none" asks for no pool CPUs. + + Args: + spec: APIC ID specification, or "none" + + Returns: + The requested APIC IDs, empty for "none" + + Raises: + ValueError: If the specification is malformed + """ + if spec_is_empty(spec, "--cpus"): + return [] + try: + return parse_cpu_spec(spec) + except ValueError as e: + raise ValueError(f"Invalid CPU specification '{spec}': {e}") from e + + +def parse_device_request(spec: Optional[str]) -> List[str]: + """ + Parse the pool device request, where "none" asks for no devices. + + Args: + spec: Comma-separated device names, "none", or None + + Returns: + The requested device names, empty for "none" and for no request + + Raises: + ValueError: If "none" is mixed with device names + """ + if not spec or spec_is_empty(spec, "--devices"): + return [] + return parse_device_list(spec) + + def validate_memory_request(requested: Dict[int, int]) -> None: """ Reject pool sizes the kernel cannot honour, however they were asked for. @@ -388,18 +455,23 @@ def parse_memory_request(spec: str) -> Dict[int, int]: "2GB" asks for 2GB without naming a node, which the caller then resolves to the node of the requested CPUs; "8GB@0,8GB@1" asks for a specific amount per node, mirroring the device-tree unit address - convention. The two forms cannot be mixed. + convention. The two forms cannot be mixed. "none", or its synonym + "0", asks for no pool memory at all. Args: spec: Memory specification string Returns: - Mapping of NUMA node id (-1 when the node is left to kerf) to size in bytes + Mapping of NUMA node id (-1 when the node is left to kerf) to size + in bytes, empty for "none" Raises: ValueError: If the specification is malformed or a size is zero or not page aligned """ + if spec_is_empty(spec, "--memory", synonyms=("0",)): + return {} + parts = [p.strip() for p in spec.split(",") if p.strip()] if not parts: raise ValueError("empty memory specification") @@ -513,10 +585,11 @@ def build_baseline_from_cmdline( Build a GlobalDeviceTree from command line arguments. Args: - cpus: CPU specification string (e.g., "4-7" or "4,5,6,7") + cpus: CPU specification string (e.g., "4-7" or "4,5,6,7"), or "none" memory: Pool memory request, either "2GB" on the node of the - requested CPUs or "8GB@0,8GB@1" for specific nodes - devices: Optional device names (comma-separated, e.g., "enp9s0_dev,nvme0") + requested CPUs, "8GB@0,8GB@1" for specific nodes, or "none" + devices: Optional device names (comma-separated, e.g., "enp9s0_dev,nvme0"), + or "none" verbose: Whether to print verbose output pool_cpus: APIC IDs the pool already holds pool_regions: Chunks the pool already holds @@ -528,11 +601,9 @@ def build_baseline_from_cmdline( ValueError: If the CPU or memory specification is invalid KernelInterfaceError: If the system topology cannot be read """ - try: - cpu_list = parse_cpu_spec(cpus) - except ValueError as e: - raise ValueError(f"Invalid CPU specification '{cpus}': {e}") from e + cpu_list = parse_cpu_request(cpus) + # Silence about memory would otherwise read as "give it all back". if not memory: raise ValueError("--memory is required") requested = parse_memory_request(memory) @@ -596,8 +667,8 @@ def build_baseline_from_cmdline( ) device_dict = {} - if devices: - device_names = parse_device_list(devices) + device_names = parse_device_request(devices) + if device_names: for device_name in device_names: device_info = detect_device_from_system(device_name) if device_info: @@ -635,24 +706,6 @@ def build_baseline_from_cmdline( return tree -def build_teardown_tree(pool_cpus: Optional[set] = None) -> GlobalDeviceTree: - """Build the empty requested state: every pool resource goes back to the host.""" - apic_ids = (get_valid_apic_ids_from_system() or set()) | set(pool_cpus or ()) - cpu_allocation = CPUAllocation( - total=(max(apic_ids) + 1) if apic_ids else 0, - host_reserved=sorted(apic_ids), - available=[] - ) - - hardware = HardwareInventory( - cpus=cpu_allocation, - memory=MemoryAllocation(total_bytes=0, host_reserved_bytes=0, requested={}), - devices={} - ) - - return GlobalDeviceTree(hardware=hardware, instances={}, device_references={}) - - INSTANCES_DIR = "/sys/fs/multikernel/instances" @@ -718,7 +771,7 @@ def read_current_pool(baseline_mgr) -> Optional[GlobalDeviceTree]: def request_is_empty(requested: GlobalDeviceTree) -> bool: - """Whether the request asks for nothing at all, as --teardown does.""" + """Whether the request asks for nothing at all, so the pool goes away.""" hardware = requested.hardware return not (hardware.cpus.available or hardware.memory.requested or hardware.devices) @@ -784,7 +837,7 @@ def reconcile_pool( Raises: KernelInterfaceError: If the kernel rejects the write or the overlay - ValidationError: If a teardown would strand running instances + ValidationError: If emptying the pool would strand running instances """ if not pool_is_live(current): # The kernel rejects a baseline with no memory@N node, and there is @@ -858,16 +911,15 @@ def _dump_baseline_dts(baseline_mgr: BaselineManager, tree: GlobalDeviceTree) -> @click.command() @click.pass_context @click.option('--input', '-i', help='Input DTS or DTB file containing all resources. Mutually exclusive with --cpus, --memory and --devices. When used, all resources must come from the file.') -@click.option('--cpus', '-c', help='APIC ID specification for baseline (e.g., "128-134" or "128,130,132"). Use physical APIC IDs, not logical CPU numbers. Mutually exclusive with --input.') -@click.option('--memory', '-m', help='Pool memory: SIZE (e.g. "2GB") on the node of the requested CPUs, or per-node "8GB@0,8GB@1". Required with --cpus, mutually exclusive with --input.') -@click.option('--devices', '-d', help='Device names (comma-separated, e.g., "enp9s0_dev,nvme0"). Mutually exclusive with --input. Creates minimal device entries in baseline.') -@click.option('--teardown', is_flag=True, help='Return every pool resource to the host. Mutually exclusive with --input, --cpus, --memory and --devices.') +@click.option('--cpus', '-c', help='APIC ID specification for baseline (e.g., "128-134" or "128,130,132"), or "none" for no pool CPUs. Use physical APIC IDs, not logical CPU numbers. Mutually exclusive with --input.') +@click.option('--memory', '-m', help='Pool memory: SIZE (e.g. "2GB") on the node of the requested CPUs, per-node "8GB@0,8GB@1", or "none" for no pool memory. Required with --cpus, mutually exclusive with --input.') +@click.option('--devices', '-d', help='Device names (comma-separated, e.g., "enp9s0_dev,nvme0"), or "none" for no devices. Mutually exclusive with --input. Creates minimal device entries in baseline.') @click.option('--dry-run', is_flag=True, help='Report the plan without applying it. Still reads the pool from the kernel, so it needs root.') -@click.option('--report', is_flag=True, help='Generate detailed validation report. Ignored with --teardown.') +@click.option('--report', is_flag=True, help='Generate detailed validation report. Ignored when the request asks for no memory.') @click.option('--format', type=click.Choice(['text', 'json', 'yaml']), default='text', help='Report format (default: text)') @click.option('--verbose', '-v', is_flag=True, help='Verbose output') -def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: Optional[str], devices: Optional[str], teardown: bool, dry_run: bool, report: bool, format: str, verbose: bool): +def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: Optional[str], devices: Optional[str], dry_run: bool, report: bool, format: str, verbose: bool): """ Initialize baseline device tree configuration. @@ -878,14 +930,18 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: The command is idempotent: the requested state is compared against the pool the kernel reports, and only the difference is applied. An empty pool takes the baseline write; a live pool is reconciled with a - /resources overlay transaction. Use --teardown to return everything to - the host, and --dry-run to see the plan without applying it. Even - --dry-run reads the pool from the kernel, so every form needs root. + /resources overlay transaction. Use --dry-run to see the plan without + applying it. Even --dry-run reads the pool from the kernel, so every + form needs root. You can either provide a DTS/DTB file via --input, or construct the request from command line arguments using --cpus and --memory. These options are mutually exclusive. + Every resource is spelled out: --cpus=none, --memory=none (or + --memory=0) and --devices=none ask for none of that resource, and a + request that asks for nothing returns the whole pool to the host. + A --memory size that names no node is placed on the NUMA node of the requested CPUs. Kerf resolves it here so the kernel is always handed an explicit node and never picks the placement itself. @@ -903,18 +959,16 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: # Shrink the pool back to 1GB and 2 CPUs kerf init --cpus=128,129 --memory=1GB + # Keep the CPUs but hand every chunk back + kerf init --cpus=128,129 --memory=none + # Return every pool resource to the host - kerf init --teardown + kerf init --cpus=none --memory=none # Show what would change without applying kerf init --cpus=128-134 --memory=1GB --dry-run """ try: - if teardown and (input or cpus or memory or devices): - click.echo("Error: --teardown cannot be combined with --input, --cpus, --memory or --devices.", err=True) - click.echo("--teardown returns every pool resource to the host.", err=True) - sys.exit(2) - # Validate that --input and resource specification options are mutually exclusive # When using --input, all resources must come from the DTS file if input and (cpus or memory or devices): @@ -930,13 +984,13 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: click.echo("Use either --input for a complete DTS/DTB file, or command-line options to construct baseline.", err=True) sys.exit(2) - if not input and not cpus and not teardown: - click.echo("Error: Either --input, --cpus or --teardown must be specified", err=True) + if not input and not cpus: + click.echo("Error: Either --input or --cpus must be specified", err=True) click.echo("\nUsage:", err=True) click.echo(" kerf init --input=hardware.dts", err=True) click.echo(" kerf init --cpus=4-7 --memory=1GB", err=True) click.echo(" kerf init --cpus=4-7 --memory=1GB@0 --devices=enp9s0_dev", err=True) - click.echo(" kerf init --teardown", err=True) + click.echo(" kerf init --cpus=none --memory=none", err=True) sys.exit(2) parser = DeviceTreeParser() @@ -949,9 +1003,7 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: live_cpus = pool_apic_ids(current) live_regions = pool_memory_regions(current) - if teardown: - tree = build_teardown_tree(pool_cpus=live_cpus) - elif input: + if input: # Parse from input file input_path = Path(input) @@ -1004,9 +1056,9 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: click.echo("\nInstances should be created via 'kerf create'", err=True) sys.exit(1) - # The teardown request holds no resources at all, which the resource - # validator reads as an unusable pool. - if not teardown: + # The resource validator reads a pool with no memory as unusable, + # which is exactly what a request for no memory asks for. + if tree.hardware.memory.requested: validator = MultikernelValidator() if dts_content is not None: input_path_str = str(input) if input else "command-line" diff --git a/src/kerf/show/main.py b/src/kerf/show/main.py index f260c15..5b11df6 100644 --- a/src/kerf/show/main.py +++ b/src/kerf/show/main.py @@ -279,9 +279,12 @@ def display_baseline_info(tree: GlobalDeviceTree, verbose: bool = False): # CPU Information click.echo("\n CPUs:") click.echo(f" Total: {hardware.cpus.total}") - click.echo( - f" Host Reserved: {len(hardware.cpus.host_reserved)} cpus: {hardware.cpus.host_reserved}" - ) + # A tree read back from the kernel describes the pool, not the host, so + # it carries no host-reserved list worth a line. + if hardware.cpus.host_reserved: + click.echo( + f" Host Reserved: {len(hardware.cpus.host_reserved)} cpus: {hardware.cpus.host_reserved}" + ) if hardware.cpus.available_free is not None: click.echo( f" Pool CPUs: {len(hardware.cpus.available)} cpus: {hardware.cpus.available}" diff --git a/tests/test_init_memory_spec.py b/tests/test_init_memory_spec.py index 565011b..d3749ac 100644 --- a/tests/test_init_memory_spec.py +++ b/tests/test_init_memory_spec.py @@ -12,11 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""The --memory specification accepted by 'kerf init'.""" +"""The resource specifications accepted by 'kerf init'.""" import pytest -from kerf.init.main import parse_memory_request, validate_memory_request +from kerf.init.main import ( + parse_cpu_request, + parse_device_request, + parse_memory_request, + validate_memory_request, +) GB = 1 << 30 @@ -38,7 +43,7 @@ def test_invalid_specs(spec): parse_memory_request(spec) -@pytest.mark.parametrize("spec", ["0", "0@0", "4097", "5000@1"]) +@pytest.mark.parametrize("spec", ["0@0", "4097", "5000@1"]) def test_sizes_must_be_positive_and_page_aligned(spec): with pytest.raises(ValueError): parse_memory_request(spec) @@ -53,3 +58,44 @@ def test_input_file_sizes_get_the_same_check(requested): def test_valid_input_file_sizes_pass(): validate_memory_request({0: GB, 1: 512 << 20}) + + +@pytest.mark.parametrize("spec", ["none", "NONE", " none ", "0"]) +def test_memory_none_asks_for_nothing(spec): + assert not parse_memory_request(spec) + + +@pytest.mark.parametrize("spec", ["none,1GB@0", "1GB@0,none", "0,2GB", "none,none"]) +def test_memory_none_cannot_be_mixed(spec): + with pytest.raises(ValueError, match="cannot be combined with other entries"): + parse_memory_request(spec) + + +@pytest.mark.parametrize("spec", ["none", "NONE", " none "]) +def test_cpus_none_asks_for_nothing(spec): + assert parse_cpu_request(spec) == [] + + +def test_cpus_are_still_parsed(): + assert parse_cpu_request("1-3,8") == [1, 2, 3, 8] + + +@pytest.mark.parametrize("spec", ["none,4", "4,none", "none,0"]) +def test_cpus_none_cannot_be_mixed(spec): + with pytest.raises(ValueError, match="cannot be combined with other entries"): + parse_cpu_request(spec) + + +@pytest.mark.parametrize("spec", ["none", "NONE", None, ""]) +def test_devices_none_asks_for_nothing(spec): + assert parse_device_request(spec) == [] + + +def test_devices_are_still_parsed(): + assert parse_device_request("enp9s0_dev, nvme0") == ["enp9s0_dev", "nvme0"] + + +@pytest.mark.parametrize("spec", ["none,nvme0", "nvme0,none"]) +def test_devices_none_cannot_be_mixed(spec): + with pytest.raises(ValueError, match="cannot be combined with other entries"): + parse_device_request(spec) diff --git a/tests/test_init_reconcile.py b/tests/test_init_reconcile.py index 1b7f088..02d7845 100644 --- a/tests/test_init_reconcile.py +++ b/tests/test_init_reconcile.py @@ -146,14 +146,14 @@ def test_apply_writes_a_pool_overlay(): assert not baseline_mgr.written -def test_teardown_returns_everything(monkeypatch): +def test_an_empty_request_returns_everything(monkeypatch): monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1, 4, 5}) monkeypatch.setattr(main, "list_instance_names", lambda: []) manager, baseline_mgr = FakeManager(), FakeBaselineManager(live=_tree([], [], {})) current = _tree([4, 5], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + requested = main.build_baseline_from_cmdline("none", memory="none") - diff = main.reconcile_pool(current, main.build_teardown_tree(), set(), False, - manager, baseline_mgr) + diff = main.reconcile_pool(current, requested, set(), False, manager, baseline_mgr) assert diff.cpus_to_host == [4, 5] assert [r.base for r in diff.memory_to_host] == [0x1_0000_0000] @@ -161,6 +161,21 @@ def test_teardown_returns_everything(monkeypatch): assert len(manager.applied) == 1 +def test_an_empty_request_dry_run_shows_the_releases(monkeypatch, capsys): + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 4, 5}) + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + current = _tree([4, 5], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + requested = main.build_baseline_from_cmdline("none", memory="none") + + main.reconcile_pool(current, requested, set(), True, manager, baseline_mgr) + out = capsys.readouterr().out + + assert "CPUs to host: 4, 5" in out + assert "Memory to host: 0x100000000 (1024 MB)" in out + assert not manager.applied + assert not baseline_mgr.written + + def test_unparseable_read_back_is_a_first_init(): manager = FakeManager() baseline_mgr = FakeBaselineManager( @@ -188,23 +203,26 @@ def test_host_cpu_list_alone_is_not_a_live_pool(): assert not manager.applied -def test_teardown_of_an_empty_pool_writes_nothing(): +def test_an_empty_request_against_an_empty_pool_writes_nothing(monkeypatch): + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1}) manager, baseline_mgr = FakeManager(), FakeBaselineManager() + requested = main.build_baseline_from_cmdline("none", memory="none") - assert reconcile_pool(_tree([0, 1], [], {}), main.build_teardown_tree(), set(), False, + assert reconcile_pool(_tree([0, 1], [], {}), requested, set(), False, manager, baseline_mgr) is None assert not baseline_mgr.written assert not manager.applied -def test_teardown_refuses_while_instances_exist(monkeypatch): +def test_an_empty_request_refuses_while_instances_exist(monkeypatch): monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 4}) monkeypatch.setattr(main, "list_instance_names", lambda: ["db", "web"]) manager, baseline_mgr = FakeManager(), FakeBaselineManager() current = _tree([4], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + requested = main.build_baseline_from_cmdline("none", memory="none") with pytest.raises(ValidationError, match="delete instances db, web first"): - reconcile_pool(current, main.build_teardown_tree(), set(), False, manager, baseline_mgr) + reconcile_pool(current, requested, set(), False, manager, baseline_mgr) assert not manager.applied @@ -277,27 +295,29 @@ def test_pool_apic_ids_ignores_a_host_only_read_back(): assert main.pool_apic_ids(_tree([0, 1, 2, 3], [], {})) == set() -def test_teardown_tree_reserves_pool_cpus(monkeypatch): +def test_an_empty_request_reserves_pool_cpus(monkeypatch): monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0}) - tree = main.build_teardown_tree(pool_cpus={1, 2, 3}) + tree = main.build_baseline_from_cmdline("none", memory="none", pool_cpus={1, 2, 3}) assert tree.hardware.cpus.host_reserved == [0, 1, 2, 3] assert not tree.hardware.cpus.available assert tree.hardware.cpus.total == 4 + assert not tree.hardware.memory.requested + assert tree.hardware.memory.total_bytes == 0 -def test_teardown_does_not_read_the_pool_back(monkeypatch, capsys): - # After a teardown /resources has no memory@N, so a read-back always - # fails; complaining about it makes a clean teardown look broken. +def test_an_empty_request_does_not_read_the_pool_back(monkeypatch, capsys): + # Once the pool is gone /resources has no memory@N, so a read-back + # always fails; complaining about it makes a clean exit look broken. monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1}) monkeypatch.setattr(main, "list_instance_names", lambda: []) manager = FakeManager() baseline_mgr = FakeBaselineManager( read_error=ParseError("No memory description in /resources")) current = _tree([1], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + requested = main.build_baseline_from_cmdline("none", memory="none", pool_cpus={1}) - main.reconcile_pool(current, main.build_teardown_tree(pool_cpus={1}), set(), False, - manager, baseline_mgr) + main.reconcile_pool(current, requested, set(), False, manager, baseline_mgr) assert "could not read the pool back" not in capsys.readouterr().err diff --git a/tests/test_show.py b/tests/test_show.py index 021709a..05e4fd4 100644 --- a/tests/test_show.py +++ b/tests/test_show.py @@ -92,8 +92,8 @@ def test_memory_total_and_host_reserved_hidden_when_zero(capsys, monkeypatch): display_baseline_info(_tree(_cpus(), memory)) out = capsys.readouterr().out - # The CPU section always prints its own Total/Host Reserved; only the - # memory ones (GB-suffixed) are suppressed when the value is 0. + # The CPU section prints its own Total and Host Reserved lines; only + # the memory ones (GB-suffixed) are suppressed when the value is 0. assert "GB) " not in out for line in out.splitlines(): assert not line.strip().startswith("Total:") or "GB" not in line @@ -147,3 +147,14 @@ def test_pool_without_chunks_says_so(capsys, monkeypatch): assert "No memory pool configured" in out assert "Chunk:" not in out + + +def test_cpu_host_reserved_hidden_when_the_kernel_reports_none(capsys, monkeypatch): + # A read-back describes the pool, not the host, and lists no reserved CPU. + cpus = CPUAllocation(total=32, host_reserved=[], available=list(range(4, 32))) + memory = MemoryAllocation(total_bytes=0, host_reserved_bytes=0, regions=[]) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(cpus, memory)) + + assert "Host Reserved:" not in capsys.readouterr().out