diff --git a/.github/workflows/lint_and_test.yml b/.github/workflows/lint_and_test.yml index 6967808..0beff83 100644 --- a/.github/workflows/lint_and_test.yml +++ b/.github/workflows/lint_and_test.yml @@ -37,6 +37,12 @@ jobs: - name: Install dependencies run: | + # pylibfdt builds its SWIG wrapper from source, and swig 4.5 dropped + # the PyInt_* macros that wrapper still uses. Poetry builds sdists + # with "pip --isolated", which ignores PIP_CONSTRAINT, so build + # pylibfdt ourselves with a pinned swig and let poetry find it. + poetry run python -m pip install "swig<4.5" setuptools setuptools_scm + poetry run python -m pip install --no-build-isolation "pylibfdt>=1.7.0,<2" poetry install --no-interaction - name: Analysing the code with pylint diff --git a/README.md b/README.md index dce2d7f..3ff27a6 100644 --- a/README.md +++ b/README.md @@ -127,11 +127,19 @@ Baseline DTB (static) ### Command Line Interface ```bash -# Initialize resource pool with CPUs (memory parsed from /proc/iomem) -kerf init --cpus=4-7 +# Initialize resource pool with CPUs and pool memory +# A plain size lands on the NUMA node of the requested CPUs +kerf init --cpus=4-7 --memory=2GB -# Initialize with CPUs and devices -kerf init --cpus=4-31 --devices=enp9s0_dev,nvme0 +# Initialize with CPUs, per-node memory and devices +kerf init --cpus=4-31 --memory=8GB@0,8GB@1 --devices=enp9s0_dev,nvme0 + +# Re-run to reshape the live pool; "none" asks for none of a resource +kerf init --cpus=4-15 --memory=4GB +kerf init --cpus=4-15 --memory=none + +# Hand everything back to the host +kerf init --cpus=none --memory=none # Create kernel instance with resource allocation kerf create web-server --cpus=4-7 --memory=2GB diff --git a/examples/bad_system.dts b/examples/bad_system.dts index 478ca89..7f113e5 100644 --- a/examples/bad_system.dts +++ b/examples/bad_system.dts @@ -14,8 +14,9 @@ resources { cpus = <4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31>; - memory-base = <0x80000000>; - memory-bytes = <0x0 0x380000000>; // 14GB + memory@0 { + size = <0x0 0x380000000>; // 14GB + }; devices { eth0: ethernet@0 { diff --git a/examples/baseline.dts b/examples/baseline.dts index 88c753e..5cce70d 100644 --- a/examples/baseline.dts +++ b/examples/baseline.dts @@ -20,8 +20,9 @@ resources { cpus = <4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31>; - memory-base = <0x80000000>; - memory-bytes = <0x0 0x380000000>; // 14GB + memory@0 { + size = <0x0 0x380000000>; // 14GB + }; devices { eth0: ethernet@0 { diff --git a/examples/conflict_example.dts b/examples/conflict_example.dts index a9cd88b..a56e87d 100644 --- a/examples/conflict_example.dts +++ b/examples/conflict_example.dts @@ -13,7 +13,8 @@ resources { cpus = <2 3 4 5 6 7 8 9 10 11 12 13 14 15>; - memory-base = <0x100000000>; - memory-bytes = <0x0 0x300000000>; // 12GB + memory@0 { + size = <0x0 0x300000000>; // 12GB + }; }; }; diff --git a/examples/edge_computing.dts b/examples/edge_computing.dts index 5e2a73f..f859d3b 100644 --- a/examples/edge_computing.dts +++ b/examples/edge_computing.dts @@ -12,8 +12,9 @@ resources { cpus = <2 3 4 5 6 7 8 9 10 11 12 13 14 15>; - memory-base = <0x200000000>; - memory-bytes = <0x0 0x600000000>; // 24GB + memory@0 { + size = <0x0 0x600000000>; // 24GB + }; devices { eth0: ethernet@0 { diff --git a/examples/high_performance.dts b/examples/high_performance.dts index 4001623..46b4783 100644 --- a/examples/high_performance.dts +++ b/examples/high_performance.dts @@ -18,8 +18,9 @@ 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119>; - memory-base = <0x400000000>; - memory-bytes = <0x0 0xC00000000>; // 48GB + memory@0 { + size = <0x0 0xC00000000>; // 48GB + }; devices { eth0: ethernet@0 { diff --git a/examples/minimal.dts b/examples/minimal.dts index ab6b2b5..10ca926 100644 --- a/examples/minimal.dts +++ b/examples/minimal.dts @@ -12,7 +12,8 @@ resources { cpus = <2 3 4 5 6 7>; - memory-base = <0x40000000>; - memory-bytes = <0x0 0x1C0000000>; // 7GB + memory@0 { + size = <0x0 0x1C0000000>; // 7GB + }; }; }; diff --git a/examples/numa_topology.dts b/examples/numa_topology.dts index 1424653..503d438 100644 --- a/examples/numa_topology.dts +++ b/examples/numa_topology.dts @@ -58,14 +58,31 @@ }; }; - memory-base = <0x0 0x800000000>; - memory-bytes = <0x0 0x1800000000>; // 96GB + memory@0 { + size = <0x0 0x600000000>; // 24GB + numa-node-id = <0>; + }; + + memory@1 { + size = <0x0 0x600000000>; // 24GB + numa-node-id = <1>; + }; + + memory@2 { + size = <0x0 0x600000000>; // 24GB + numa-node-id = <2>; + }; + + memory@3 { + size = <0x0 0x600000000>; // 24GB + numa-node-id = <3>; + }; devices { eth0: ethernet@0 { compatible = "intel,i40e"; pci-id = "0000:01:00.0"; - numa-node = <0>; + numa-node-id = <0>; sriov-vfs = <8>; host-reserved-vf = <0>; available-vfs = <1 2 3 4 5 6 7>; @@ -74,7 +91,7 @@ eth1: ethernet@1 { compatible = "intel,i40e"; pci-id = "0000:02:00.0"; - numa-node = <2>; + numa-node-id = <2>; sriov-vfs = <8>; host-reserved-vf = <0>; available-vfs = <1 2 3 4 5 6 7>; @@ -83,7 +100,7 @@ nvme0: storage@0 { compatible = "nvme"; pci-id = "0000:03:00.0"; - numa-node = <0>; + numa-node-id = <0>; namespaces = <4>; host-reserved-ns = <1>; available-ns = <2 3 4>; @@ -92,7 +109,7 @@ nvme1: storage@1 { compatible = "nvme"; pci-id = "0000:04:00.0"; - numa-node = <2>; + numa-node-id = <2>; namespaces = <4>; host-reserved-ns = <1>; available-ns = <2 3 4>; diff --git a/examples/simple_numa.dts b/examples/simple_numa.dts index 59e08fa..fcf1c3b 100644 --- a/examples/simple_numa.dts +++ b/examples/simple_numa.dts @@ -32,8 +32,15 @@ }; }; - memory-base = <0x0 0x400000000>; - memory-bytes = <0x0 0xC00000000>; // 48GB + memory@0 { + size = <0x0 0x600000000>; // 24GB + numa-node-id = <0>; + }; + + memory@1 { + size = <0x0 0x600000000>; // 24GB + numa-node-id = <1>; + }; devices { eth0: ethernet@0 { diff --git a/examples/system.dts b/examples/system.dts index 96b89e7..c540734 100644 --- a/examples/system.dts +++ b/examples/system.dts @@ -13,8 +13,9 @@ resources { cpus = <4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31>; - memory-base = <0x80000000>; - memory-bytes = <0x0 0x380000000>; // 14GB + memory@0 { + size = <0x0 0x380000000>; // 14GB + }; devices { eth0: ethernet@0 { diff --git a/src/kerf/__init__.py b/src/kerf/__init__.py index 6e21e19..bd3e205 100644 --- a/src/kerf/__init__.py +++ b/src/kerf/__init__.py @@ -41,7 +41,7 @@ get_available_cpus, get_allocated_cpus, get_allocated_memory_regions, - find_available_memory_base, + chunk_containing, validate_cpu_allocation, validate_memory_allocation, find_next_instance_id, @@ -67,7 +67,7 @@ "get_available_cpus", "get_allocated_cpus", "get_allocated_memory_regions", - "find_available_memory_base", + "chunk_containing", "validate_cpu_allocation", "validate_memory_allocation", "find_next_instance_id", diff --git a/src/kerf/create/main.py b/src/kerf/create/main.py index 6ec93be..568949f 100644 --- a/src/kerf/create/main.py +++ b/src/kerf/create/main.py @@ -31,7 +31,6 @@ from ..resources import ( validate_cpu_allocation, validate_memory_allocation, - find_available_memory_base, get_available_cpus, ) from ..exceptions import ValidationError, KernelInterfaceError, ResourceError, ParseError @@ -302,6 +301,12 @@ def parse_memory_spec(memory_spec: str) -> int: ) from exc +def _placement(instance) -> str: + """Name the base only when one was asked for; the kernel picks otherwise.""" + base = instance.resources.memory_base + return f" at {hex(base)}" if base else "" + + def parse_memory_base(base_spec: str) -> int: """ Parse memory base address specification. @@ -432,7 +437,8 @@ def dump_overlay_for_debug( ) @click.option( "--memory-base", - help="Memory base address (hex: 0x80000000 or decimal, auto-assigned if not specified)", + help="Memory base address to request (hex: 0x80000000 or decimal). " + "Only checked against the pool; the kernel places instance memory itself.", ) @click.option( "--devices", @@ -641,17 +647,11 @@ def create_instance_operation(current): # Validate CPU allocation (against baseline and existing instances) validate_cpu_allocation(modified, cpu_list) - # Find memory base if not specified + # The kernel places instance memory itself; a base is only ever + # a caller's request, so it is all there is to validate. if memory_base_addr is None: - found_base = find_available_memory_base(modified, memory_bytes) - if found_base is None: - raise ResourceError( - f"No available memory region found for {memory_bytes} bytes. " - "Try specifying --memory-base or reduce memory size." - ) - memory_base_addr = found_base + memory_base_addr = 0 else: - # Validate specified memory base validate_memory_allocation(modified, memory_base_addr, memory_bytes) # Create instance resources with topology settings @@ -705,7 +705,7 @@ def create_instance_operation(current): click.echo( f" NUMA Nodes: {', '.join(map(str, instance.resources.numa_nodes))}" ) - click.echo(f" Memory: {memory} at {hex(instance.resources.memory_base)}") + click.echo(f" Memory: {memory}{_placement(instance)}") if instance.resources.memory_policy: click.echo(f" Memory Policy: {instance.resources.memory_policy}") if instance.resources.devices: @@ -745,7 +745,7 @@ def create_instance_operation(current): if instance.resources.numa_nodes: numa_str = ", ".join(map(str, instance.resources.numa_nodes)) click.echo(f" NUMA Nodes: {numa_str}") - click.echo(f" Memory: {memory} at {hex(instance.resources.memory_base)}") + click.echo(f" Memory: {memory}{_placement(instance)}") if instance.resources.memory_policy: click.echo(f" Memory Policy: {instance.resources.memory_policy}") if instance.resources.devices: diff --git a/src/kerf/dtc/extractor.py b/src/kerf/dtc/extractor.py index c7bfb0d..f45055b 100644 --- a/src/kerf/dtc/extractor.py +++ b/src/kerf/dtc/extractor.py @@ -18,6 +18,7 @@ import libfdt from ..models import GlobalDeviceTree, Instance +from ..pool_diff import ANY_NODE from .cells import pack_cpu_ids @@ -125,9 +126,14 @@ def _add_cpu_properties_sw(self, fdt_sw, cpus): fdt_sw.property("cpus", pack_cpu_ids(cpus.available)) def _add_memory_properties_sw(self, fdt_sw, memory): - """Add memory properties directly to resources node.""" - fdt_sw.property_u64("memory-base", memory.memory_pool_base) - fdt_sw.property_u64("memory-bytes", memory.memory_pool_bytes) + """Add one memory@ request node per requested size (or live chunk).""" + entries = list(memory.requested.items()) or [(r.node, r.size) for r in memory.regions] + for idx, (node, size) in enumerate(entries): + fdt_sw.begin_node(f"memory@{idx}") + fdt_sw.property_u64("size", size) + if node != ANY_NODE: + fdt_sw.property_u32("numa-node-id", node) + fdt_sw.end_node() def _add_devices_section_sw(self, fdt_sw, devices): """Add devices section using FdtSw.""" @@ -244,78 +250,6 @@ def _add_device_references_sw(self, fdt_sw, device_references): fdt_sw.end_node() - def _add_resources_section(self, parent_offset: int, tree: GlobalDeviceTree): - """Add resources section to DTB.""" - resources_offset = self.fdt.add_subnode(parent_offset, "resources") - - # Add CPU information - self._add_cpu_section(resources_offset, tree.hardware.cpus) - - # Add memory information - self._add_memory_section(resources_offset, tree.hardware.memory) - - # Add device information - self._add_devices_section(resources_offset, tree.hardware.devices) - - def _add_cpu_section(self, parent_offset: int, cpus): - """Add CPU section to DTB.""" - cpus_offset = self.fdt.add_subnode(parent_offset, "cpus") - self.fdt.setprop_u32(cpus_offset, "total", cpus.total) - - self.fdt.setprop(cpus_offset, "host-reserved", pack_cpu_ids(cpus.host_reserved)) - self.fdt.setprop(cpus_offset, "available", pack_cpu_ids(cpus.available)) - - def _add_memory_section(self, parent_offset: int, memory): - """Add memory section to DTB.""" - memory_offset = self.fdt.add_subnode(parent_offset, "memory") - self.fdt.setprop_u64(memory_offset, "total-bytes", memory.total_bytes) - self.fdt.setprop_u64(memory_offset, "host-reserved-bytes", memory.host_reserved_bytes) - self.fdt.setprop_u64(memory_offset, "memory-pool-base", memory.memory_pool_base) - self.fdt.setprop_u64(memory_offset, "memory-pool-bytes", memory.memory_pool_bytes) - - def _add_devices_section(self, parent_offset: int, devices): - """Add devices section to DTB.""" - devices_offset = self.fdt.add_subnode(parent_offset, "devices") - - for name, device_info in devices.items(): - device_offset = self.fdt.add_subnode(devices_offset, name) - self.fdt.setprop_str(device_offset, "compatible", device_info.compatible) - - if device_info.pci_id: - self.fdt.setprop_str(device_offset, "pci-id", device_info.pci_id) - - if device_info.sriov_vfs is not None: - self.fdt.setprop_u32(device_offset, "sriov-vfs", device_info.sriov_vfs) - - if device_info.host_reserved_vf is not None: - self.fdt.setprop_u32( - device_offset, "host-reserved-vf", device_info.host_reserved_vf - ) - - if device_info.available_vfs: - import struct - - vfs_data = struct.pack( - ">" + "I" * len(device_info.available_vfs), *device_info.available_vfs - ) - self.fdt.setprop(device_offset, "available-vfs", vfs_data) - - if device_info.namespaces is not None: - self.fdt.setprop_u32(device_offset, "namespaces", device_info.namespaces) - - if device_info.host_reserved_ns is not None: - self.fdt.setprop_u32( - device_offset, "host-reserved-ns", device_info.host_reserved_ns - ) - - if device_info.available_ns: - import struct - - ns_data = struct.pack( - ">" + "I" * len(device_info.available_ns), *device_info.available_ns - ) - self.fdt.setprop(device_offset, "available-ns", ns_data) - def _add_instances_section(self, parent_offset: int, tree: GlobalDeviceTree): """Add instances section to DTB.""" instances_offset = self.fdt.add_subnode(parent_offset, "instances") diff --git a/src/kerf/dtc/overlay.py b/src/kerf/dtc/overlay.py index a0e6829..d9dbc3f 100644 --- a/src/kerf/dtc/overlay.py +++ b/src/kerf/dtc/overlay.py @@ -19,17 +19,35 @@ overlays (DTBO) that represent incremental changes to the device tree state. """ -from typing import Set +import struct +from typing import Optional, Set, Tuple import libfdt from ..models import GlobalDeviceTree +from ..pool_diff import ANY_NODE, PoolDiff from .cells import pack_cpu_id, pack_cpu_ids +Range = Optional[Tuple[int, int]] + + +def _memory_ranges(old_base: int, old_size: int, new_base: int, new_size: int) -> Tuple[Range, Range]: + """Ranges to take back from and to hand to an instance whose memory changed.""" + if old_base != new_base: + return (old_base, old_size), (new_base, new_size) + if new_size > old_size: + return None, (old_base + old_size, new_size - old_size) + if new_size < old_size: + return (old_base + new_size, old_size - new_size), None + return None, None + class OverlayGenerator: """Generates device tree overlay blobs (DTBO) from device tree model deltas.""" + POOL_PATH = "/resources" + INSTANCES_PATH = "/instances" + def generate_overlay(self, current: GlobalDeviceTree, modified: GlobalDeviceTree) -> bytes: """ Generate overlay DTBO representing the difference between current and modified states. @@ -92,8 +110,6 @@ def generate_update_overlay(self, instance_name: str, old_instance, new_instance Returns: DTBO blob as bytes containing resource update operations """ - import struct - fdt_sw = libfdt.FdtSw() fdt_sw.finish_reservemap() @@ -105,11 +121,16 @@ def generate_update_overlay(self, instance_name: str, old_instance, new_instance cpus_to_remove = sorted(old_cpus - new_cpus) cpus_to_add = sorted(new_cpus - old_cpus) - old_mem_base = old_instance.resources.memory_base - old_mem_size = old_instance.resources.memory_bytes - new_mem_base = new_instance.resources.memory_base - new_mem_size = new_instance.resources.memory_bytes - memory_changed = (old_mem_base != new_mem_base) or (old_mem_size != new_mem_size) + memory_to_remove, memory_to_add = _memory_ranges( + old_instance.resources.memory_base, + old_instance.resources.memory_bytes, + new_instance.resources.memory_base, + new_instance.resources.memory_bytes, + ) + + numa_node = None + if new_instance.resources.numa_nodes: + numa_node = new_instance.resources.numa_nodes[0] old_devices = set(old_instance.resources.devices) new_devices = set(new_instance.resources.devices) @@ -118,160 +139,121 @@ def generate_update_overlay(self, instance_name: str, old_instance, new_instance # Single fragment with all operations fdt_sw.begin_node("fragment@0") + fdt_sw.property_string("target-path", f"{self.INSTANCES_PATH}/{instance_name}") fdt_sw.begin_node("__overlay__") - # 1. memory-remove (if memory shrunk or base changed) - if memory_changed: - if old_mem_base == new_mem_base: - # Same base: only remove the excess if shrinking - if old_mem_size > new_mem_size: - remove_base = old_mem_base + new_mem_size - remove_size = old_mem_size - new_mem_size - fdt_sw.begin_node("memory-remove") - fdt_sw.property_string("mk,instance", instance_name) - fdt_sw.begin_node("region@0") - reg_data = struct.pack(">QQ", remove_base, remove_size) - fdt_sw.property("reg", reg_data) - fdt_sw.end_node() - fdt_sw.end_node() - else: - # Different base: remove entire old region - fdt_sw.begin_node("memory-remove") - fdt_sw.property_string("mk,instance", instance_name) - fdt_sw.begin_node("region@0") - reg_data = struct.pack(">QQ", old_mem_base, old_mem_size) - fdt_sw.property("reg", reg_data) - fdt_sw.end_node() - fdt_sw.end_node() + if memory_to_remove: + fdt_sw.begin_node("memory-remove") + self._memory_item(fdt_sw, memory_to_remove) + fdt_sw.end_node() - # 2. memory-add (if memory grew or base changed) - if memory_changed: - if old_mem_base == new_mem_base: - # Same base: only add the extension if growing - if new_mem_size > old_mem_size: - add_base = old_mem_base + old_mem_size - add_size = new_mem_size - old_mem_size - fdt_sw.begin_node("memory-add") - fdt_sw.property_string("mk,instance", instance_name) - fdt_sw.begin_node("region@0") - reg_data = struct.pack(">QQ", add_base, add_size) - fdt_sw.property("reg", reg_data) - fdt_sw.end_node() - fdt_sw.end_node() - else: - # Different base: add entire new region - fdt_sw.begin_node("memory-add") - fdt_sw.property_string("mk,instance", instance_name) - fdt_sw.begin_node("region@0") - reg_data = struct.pack(">QQ", new_mem_base, new_mem_size) - fdt_sw.property("reg", reg_data) - fdt_sw.end_node() - fdt_sw.end_node() + if memory_to_add: + fdt_sw.begin_node("memory-add") + self._memory_item(fdt_sw, memory_to_add, numa_node) + fdt_sw.end_node() - # 3. cpu-remove (if CPUs removed) - if cpus_to_remove: - fdt_sw.begin_node("cpu-remove") - fdt_sw.property_string("mk,instance", instance_name) + self._cpu_op(fdt_sw, "cpu-remove", cpus_to_remove) + self._cpu_op(fdt_sw, "cpu-add", cpus_to_add, numa_node) + self._device_op(fdt_sw, "device-remove", devices_to_remove) + self._device_op(fdt_sw, "device-add", devices_to_add) - for cpu_id in cpus_to_remove: - fdt_sw.begin_node(f"cpu@{cpu_id}") - fdt_sw.property("reg", pack_cpu_id(cpu_id)) - fdt_sw.end_node() + fdt_sw.end_node() # End __overlay__ + fdt_sw.end_node() # End fragment@0 - fdt_sw.end_node() + fdt_sw.end_node() # End root - # 4. cpu-add (if CPUs added) - if cpus_to_add: - fdt_sw.begin_node("cpu-add") - fdt_sw.property_string("mk,instance", instance_name) + dtb = fdt_sw.as_fdt() + dtb.pack() + return dtb.as_bytearray() - for cpu_id in cpus_to_add: - fdt_sw.begin_node(f"cpu@{cpu_id}") - fdt_sw.property("reg", pack_cpu_id(cpu_id)) + def generate_pool_overlay(self, diff: PoolDiff) -> bytes: + """ + Generate an overlay moving resources between the host and the pool. - if new_instance.resources.numa_nodes: - fdt_sw.property_u32("numa-node", new_instance.resources.numa_nodes[0]) + Operation names read from the pool's point of view: memory-add grows the + pool, cpu-remove returns a pool CPU to the host, and so on. - fdt_sw.end_node() + Args: + diff: Resources to move, as computed against the requested baseline - fdt_sw.end_node() + Returns: + DTBO blob as bytes containing a single fragment targeting /resources + """ + fdt_sw = libfdt.FdtSw() + fdt_sw.finish_reservemap() - # 5. device-remove (if devices removed) - if devices_to_remove: - fdt_sw.begin_node("device-remove") - fdt_sw.property_string("mk,instance", instance_name) + fdt_sw.begin_node("") + fdt_sw.property_string("compatible", "linux,multikernel-overlay") - for idx, pci_id in enumerate(devices_to_remove): - fdt_sw.begin_node(f"pci@{idx}") - fdt_sw.property_string("pci-id", pci_id) - fdt_sw.end_node() + fdt_sw.begin_node("fragment@0") + fdt_sw.property_string("target-path", self.POOL_PATH) + fdt_sw.begin_node("__overlay__") + if diff.memory_to_host: + fdt_sw.begin_node("memory-remove") + for idx, region in enumerate(diff.memory_to_host): + fdt_sw.begin_node(f"memory@{idx}") + fdt_sw.property("reg", struct.pack(">QQ", region.base, region.size)) + fdt_sw.end_node() fdt_sw.end_node() - # 6. device-add (if devices added) - if devices_to_add: - fdt_sw.begin_node("device-add") - fdt_sw.property_string("mk,instance", instance_name) - - for idx, pci_id in enumerate(devices_to_add): - fdt_sw.begin_node(f"pci@{idx}") - fdt_sw.property_string("pci-id", pci_id) + if diff.memory_to_pool: + fdt_sw.begin_node("memory-add") + for idx, (node, size) in enumerate(diff.memory_to_pool): + fdt_sw.begin_node(f"memory@{idx}") + fdt_sw.property_u64("size", size) + if node != ANY_NODE: + fdt_sw.property_u32("numa-node-id", node) fdt_sw.end_node() - fdt_sw.end_node() + self._cpu_op(fdt_sw, "cpu-remove", diff.cpus_to_host) + self._cpu_op(fdt_sw, "cpu-add", diff.cpus_to_pool) + self._device_op(fdt_sw, "device-remove", diff.devices_to_host) + self._device_op(fdt_sw, "device-add", diff.devices_to_pool) + fdt_sw.end_node() # End __overlay__ fdt_sw.end_node() # End fragment@0 - fdt_sw.end_node() # End root dtb = fdt_sw.as_fdt() dtb.pack() return dtb.as_bytearray() - def _add_memory_operation(self, fdt_sw, fragment_id, operation, instance_name, base, size): - """Helper to add memory operation fragment.""" - import struct - - fdt_sw.begin_node(f"fragment@{fragment_id}") - fdt_sw.begin_node("__overlay__") - fdt_sw.begin_node(operation) - fdt_sw.property_string("mk,instance", instance_name) - - fdt_sw.begin_node("region@0") - reg_data = struct.pack(">QQ", base, size) - fdt_sw.property("reg", reg_data) + def _memory_item(self, fdt_sw, region, numa_node=None): + """Write a memory@0 item naming an existing range.""" + base, size = region + fdt_sw.begin_node("memory@0") + fdt_sw.property("reg", struct.pack(">QQ", base, size)) + if numa_node is not None: + fdt_sw.property_u32("numa-node-id", numa_node) fdt_sw.end_node() - fdt_sw.end_node() - fdt_sw.end_node() - fdt_sw.end_node() - - return fragment_id + 1 + def _cpu_op(self, fdt_sw, operation, cpu_ids, numa_node=None): + """Write a CPU operation node, or nothing when there are no CPUs.""" + if not cpu_ids: + return - def _add_cpu_operation( - self, fdt_sw, fragment_id, operation, instance_name, cpu_ids, numa_nodes - ): - """Helper to add CPU operation fragment.""" - fdt_sw.begin_node(f"fragment@{fragment_id}") - fdt_sw.begin_node("__overlay__") fdt_sw.begin_node(operation) - fdt_sw.property_string("mk,instance", instance_name) - for cpu_id in cpu_ids: fdt_sw.begin_node(f"cpu@{cpu_id}") fdt_sw.property("reg", pack_cpu_id(cpu_id)) + if numa_node is not None: + fdt_sw.property_u32("numa-node-id", numa_node) + fdt_sw.end_node() + fdt_sw.end_node() - if operation == "cpu-add" and numa_nodes: - fdt_sw.property_u32("numa-node", numa_nodes[0]) + def _device_op(self, fdt_sw, operation, pci_ids): + """Write a device operation node, or nothing when there are no devices.""" + if not pci_ids: + return + fdt_sw.begin_node(operation) + for idx, pci_id in enumerate(pci_ids): + fdt_sw.begin_node(f"pci@{idx}") + fdt_sw.property_string("pci-id", pci_id) fdt_sw.end_node() - - fdt_sw.end_node() fdt_sw.end_node() - fdt_sw.end_node() - - return fragment_id + 1 def _create_overlay_dtb( self, instances_to_add: dict, instances_to_update: dict, instances_to_remove: Set[str] @@ -298,7 +280,8 @@ def _create_overlay_dtb( all_instances = {**instances_to_add, **instances_to_update} for name, instance in all_instances.items(): - fdt_sw.begin_node(f"fragment@{fragment_id}") + fdt_sw.begin_node(f"fragment@{fragment_id:x}") + fdt_sw.property_string("target-path", self.INSTANCES_PATH) fdt_sw.begin_node("__overlay__") fdt_sw.begin_node("instance-create") @@ -311,7 +294,6 @@ def _create_overlay_dtb( fdt_sw.property("cpus", pack_cpu_ids(instance.resources.cpus)) - fdt_sw.property_u64("memory-base", instance.resources.memory_base) fdt_sw.property_u64("memory-bytes", instance.resources.memory_bytes) if instance.resources.devices: @@ -319,8 +301,6 @@ def _create_overlay_dtb( fdt_sw.property("device-names", stringlist_data) if instance.resources.numa_nodes: - import struct - numa_data = struct.pack( ">" + "I" * len(instance.resources.numa_nodes), *instance.resources.numa_nodes ) @@ -362,7 +342,8 @@ def _create_overlay_dtb( fragment_id += 1 for name in instances_to_remove: - fdt_sw.begin_node(f"fragment@{fragment_id}") + fdt_sw.begin_node(f"fragment@{fragment_id:x}") + fdt_sw.property_string("target-path", self.INSTANCES_PATH) fdt_sw.begin_node("__overlay__") fdt_sw.begin_node("instance-remove") fdt_sw.property_string("instance-name", name) diff --git a/src/kerf/dtc/parser.py b/src/kerf/dtc/parser.py index 0184739..0c95131 100644 --- a/src/kerf/dtc/parser.py +++ b/src/kerf/dtc/parser.py @@ -17,11 +17,13 @@ """ import re -from typing import Dict, List, Optional +import struct +from typing import Dict, List, Optional, Tuple import libfdt from ..exceptions import ParseError +from ..pool_diff import ANY_NODE from .cells import unpack_cpu_ids from ..models import ( CPUAllocation, @@ -33,10 +35,25 @@ MemoryAllocation, NUMANode, OverlayInstanceData, + PoolMemoryRegion, TopologySection, ) +_LEGACY_MEMORY_ERROR = ( + "memory-base/memory-bytes are not supported; " + "use memory@N { size; numa-node-id; }" +) + +_NO_MEMORY_ERROR = ( + "No memory description in /resources; " + "expected memory@N { size; numa-node-id; }" +) + +# A live pool always publishes cpus-available, and may legitimately hold no +# memory at all: a transaction that gave back every chunk but kept the CPUs. + + class DeviceTreeParser: """Parser for multikernel device trees.""" @@ -138,8 +155,6 @@ def _build_global_tree(self) -> GlobalDeviceTree: memory=MemoryAllocation( total_bytes=0, host_reserved_bytes=0, - memory_pool_base=0, - memory_pool_bytes=0 ), topology=None, devices={} @@ -210,6 +225,11 @@ def _parse_cpu_allocation(self, resources_node: int) -> CPUAllocation: # No cpus property means all CPUs are allocated available = [] + available_free = None + free_prop = self._optional_prop(resources_node, 'cpus-available') + if free_prop is not None: + available_free = unpack_cpu_ids(free_prop) + if available: total = max(available) + 1 else: @@ -219,35 +239,76 @@ def _parse_cpu_allocation(self, resources_node: int) -> CPUAllocation: return CPUAllocation( total=total, host_reserved=host_reserved, - available=available + available=available, + available_free=available_free ) - def _parse_memory_allocation(self, resources_node: int) -> MemoryAllocation: - """Parse memory allocation from resources node.""" + def _subnodes(self, node: int): + """Yield the offsets of a node's immediate children.""" try: - memory_pool_base_prop = self.fdt.getprop(resources_node, 'memory-base') - if len(memory_pool_base_prop) != 8: - raise ParseError(f"Invalid 'memory-base' property size: {len(memory_pool_base_prop)} bytes (expected 8 bytes)") - memory_pool_base = memory_pool_base_prop.as_uint64() - except libfdt.FdtException as e: - raise ParseError(f"Missing 'memory-base' property in /resources: {e}") from e + offset = self.fdt.first_subnode(node) + except libfdt.FdtException: + return + while offset >= 0: + yield offset + try: + offset = self.fdt.next_subnode(offset) + except libfdt.FdtException: + return + def _optional_prop(self, node: int, name: str): + """Return a property, or None when the node does not carry it.""" try: - memory_pool_bytes_prop = self.fdt.getprop(resources_node, 'memory-bytes') - if len(memory_pool_bytes_prop) != 8: - raise ParseError(f"Invalid 'memory-bytes' property size: {len(memory_pool_bytes_prop)} bytes (expected 8 bytes)") - memory_pool_bytes = memory_pool_bytes_prop.as_uint64() - except libfdt.FdtException as e: - raise ParseError(f"Missing 'memory-bytes' property in /resources: {e}") from e + return self.fdt.getprop(node, name) + except libfdt.FdtException: + return None + + def _optional_u32(self, node: int, name: str, default: int) -> int: + """Return a u32 property, or default when the node does not carry it.""" + prop = self._optional_prop(node, name) + if prop is None: + return default + return prop.as_uint32() + + def _parse_memory_allocation(self, resources_node: int) -> MemoryAllocation: + """Parse pool chunks and per-node memory requests from resources node.""" + regions = [] + requested = {} + + for node in self._subnodes(resources_node): + name = self.fdt.get_name(node) + if not name.startswith('memory@'): + continue + node_id = self._optional_u32(node, 'numa-node-id', ANY_NODE) + reg = self._optional_prop(node, 'reg') + size_prop = self._optional_prop(node, 'size') + if reg is not None: + if len(reg) != 16: + raise ParseError(f"{name}: 'reg' must be two u64 cells") + base, size = struct.unpack('>QQ', bytes(reg)) + regions.append(PoolMemoryRegion(base=base, size=size, node=node_id)) + elif size_prop is not None: + if len(size_prop) != 8: + raise ParseError(f"{name}: 'size' must be a u64") + requested[node_id] = requested.get(node_id, 0) + size_prop.as_uint64() + else: + raise ParseError(f"{name}: expected 'reg' (existing chunk) or 'size' (request)") + + for legacy in ('memory-base', 'memory-bytes'): + if self._optional_prop(resources_node, legacy) is not None: + raise ParseError(_LEGACY_MEMORY_ERROR) - total_bytes = memory_pool_base + memory_pool_bytes - host_reserved_bytes = 0 + if not regions and not requested: + if self._optional_prop(resources_node, 'cpus-available') is None: + raise ParseError(_NO_MEMORY_ERROR) + + total_bytes = sum(r.size for r in regions) or sum(requested.values()) return MemoryAllocation( total_bytes=total_bytes, - host_reserved_bytes=host_reserved_bytes, - memory_pool_base=memory_pool_base, - memory_pool_bytes=memory_pool_bytes + host_reserved_bytes=0, + regions=regions, + requested=requested ) def _parse_devices(self, resources_node: int) -> Dict[str, DeviceInfo]: @@ -758,6 +819,12 @@ def _parse_cpus_from_dts(self, dts_content: str) -> CPUAllocation: raise ParseError("Missing 'cpus' property in /resources") available = [int(x.strip()) for x in cpus_match.group(1).split()] + + free_match = re.search(r'cpus-available\s*=\s*<([^>]+)>', resources_text) + available_free = None + if free_match: + available_free = [int(x.strip()) for x in free_match.group(1).split()] + if available: total = max(available) + 1 else: @@ -771,35 +838,90 @@ def _parse_cpus_from_dts(self, dts_content: str) -> CPUAllocation: total=total, host_reserved=host_reserved, available=available, - topology=topology + topology=topology, + available_free=available_free ) + def _split_node_body(self, body: str) -> Tuple[str, List[Tuple[str, str]]]: + """Split a DTS node body into its own property text and its direct children. + + Nested nodes carry properties of their own (a NUMA node@N describes itself + with memory-base/memory-size), so callers must not confuse them with the + properties of the node they are inspecting. + """ + properties = [] + children = [] + depth = 0 + chunk_start = 0 + body_start = 0 + name = "" + + for i, char in enumerate(body): + if char == '{': + depth += 1 + if depth == 1: + head = body[chunk_start:i] + tokens = head.replace(';', ' ').split() + name = tokens[-1] if tokens else "" + properties.append(head[:head.rfind(name)] if name else head) + body_start = i + 1 + elif char == '}': + depth -= 1 + if depth == 0: + children.append((name, body[body_start:i])) + chunk_start = i + 1 + + properties.append(body[chunk_start:]) + return '\n'.join(properties), children + def _parse_memory_from_dts(self, dts_content: str) -> MemoryAllocation: - """Parse memory allocation from DTS content.""" + """Parse pool chunks and per-node memory requests from DTS content.""" resources_text = self._extract_resources_section(dts_content) if not resources_text: raise ParseError("Missing /resources section in DTS") - memory_base_match = re.search(r'memory-base\s*=\s*<([^>]+)>', resources_text) - if not memory_base_match: - raise ParseError("Missing 'memory-base' property in /resources") + own_properties, children = self._split_node_body(resources_text) + if re.search(r'memory-(?:base|bytes)\s*=', own_properties): + raise ParseError(_LEGACY_MEMORY_ERROR) - memory_bytes_match = re.search(r'memory-bytes\s*=\s*<([^>]+)>', resources_text) - if not memory_bytes_match: - raise ParseError("Missing 'memory-bytes' property in /resources") + regions = [] + requested = {} + + for name, body in children: + if not name.startswith('memory@'): + continue + node_id = ANY_NODE + node_match = re.search(r'numa-node-id\s*=\s*<\s*([^>\s]+)\s*>', body) + if node_match: + node_id = int(node_match.group(1), 0) + + reg_match = re.search(r'reg\s*=\s*<([^>]+)>', body) + size_match = re.search(r'(?]+)>', body) + if reg_match: + cells = reg_match.group(1).split() + if len(cells) != 4: + raise ParseError(f"{name}: 'reg' must be two u64 cells") + base = self._parse_hex_value(' '.join(cells[:2])) + size = self._parse_hex_value(' '.join(cells[2:])) + regions.append(PoolMemoryRegion(base=base, size=size, node=node_id)) + elif size_match: + size = self._parse_hex_value(size_match.group(1)) + requested[node_id] = requested.get(node_id, 0) + size + else: + raise ParseError(f"{name}: expected 'reg' (existing chunk) or 'size' (request)") - memory_pool_base = self._parse_hex_value(memory_base_match.group(1)) - memory_pool_bytes = self._parse_hex_value(memory_bytes_match.group(1)) + if not regions and not requested: + if not re.search(r'cpus-available\s*=', own_properties): + raise ParseError(_NO_MEMORY_ERROR) - total_bytes = memory_pool_base + memory_pool_bytes - host_reserved_bytes = 0 + total_bytes = sum(r.size for r in regions) or sum(requested.values()) return MemoryAllocation( total_bytes=total_bytes, - host_reserved_bytes=host_reserved_bytes, - memory_pool_base=memory_pool_base, - memory_pool_bytes=memory_pool_bytes + host_reserved_bytes=0, + regions=regions, + requested=requested ) def _parse_devices_from_dts(self, dts_content: str) -> Dict[str, DeviceInfo]: diff --git a/src/kerf/dtc/reporter.py b/src/kerf/dtc/reporter.py index 3becb16..54ee2f3 100644 --- a/src/kerf/dtc/reporter.py +++ b/src/kerf/dtc/reporter.py @@ -123,13 +123,18 @@ def _format_hardware_inventory(self, tree: GlobalDeviceTree) -> List[str]: total_gb = memory.total_bytes / (1024**3) host_gb = memory.host_reserved_bytes / (1024**3) memory_pool_gb = memory.memory_pool_bytes / (1024**3) - host_mem_percent = (memory.host_reserved_bytes / memory.total_bytes) * 100 - memory_pool_mem_percent = (memory.memory_pool_bytes / memory.total_bytes) * 100 + # total_bytes is 0 when the tree only carries pool chunks read back + # from the kernel, not a system-wide total. + if memory.total_bytes: + host_mem_percent = f"{(memory.host_reserved_bytes / memory.total_bytes) * 100:.0f}%" + memory_pool_mem_percent = f"{(memory.memory_pool_bytes / memory.total_bytes) * 100:.0f}%" + else: + host_mem_percent = memory_pool_mem_percent = "n/a" lines.append(f" Memory: {total_gb:.0f}GB total") - lines.append(f" Host reserved: {host_gb:.0f}GB ({host_mem_percent:.0f}%)") + lines.append(f" Host reserved: {host_gb:.0f}GB ({host_mem_percent})") lines.append( - f" Memory pool: {memory_pool_gb:.0f}GB at {hex(memory.memory_pool_base)} ({memory_pool_mem_percent:.0f}%)" + f" Memory pool: {memory_pool_gb:.0f}GB at {hex(memory.memory_pool_base)} ({memory_pool_mem_percent})" ) # Device information diff --git a/src/kerf/dtc/validator.py b/src/kerf/dtc/validator.py index 6bcda48..d5e46d3 100644 --- a/src/kerf/dtc/validator.py +++ b/src/kerf/dtc/validator.py @@ -20,7 +20,7 @@ from pathlib import Path from typing import List, Dict, Optional from ..models import GlobalDeviceTree, ValidationResult, ResourceUsage -from ..resources import get_memory_pool_from_iomem +from ..resources import chunk_containing, get_pool_chunks_from_iomem class MultikernelValidator: @@ -254,47 +254,32 @@ def _validate_hardware_inventory(self, tree: GlobalDeviceTree): ) memory = tree.hardware.memory - if memory.total_bytes <= 0: - self.errors.append("Hardware inventory: Total memory must be positive") - if memory.memory_pool_bytes <= 0: self.errors.append("Hardware inventory: Spawn pool size must be positive") - iomem_pool = get_memory_pool_from_iomem() - if iomem_pool is None or ( - memory.memory_pool_base != iomem_pool[0] or memory.memory_pool_bytes != iomem_pool[1] - ): - # Pool doesn't match kernel-provided pool, validate against total_bytes - if memory.memory_pool_base + memory.memory_pool_bytes > memory.total_bytes: - self.errors.append("Hardware inventory: Spawn pool extends beyond total memory") - if iomem_pool is not None: - iomem_base, iomem_size = iomem_pool - iomem_end = iomem_base + iomem_size - - if memory.memory_pool_base != iomem_base: - self.errors.append( - f"Hardware inventory: Memory pool base ({hex(memory.memory_pool_base)}) " - f"does not match multikernel pool in /proc/iomem ({hex(iomem_base)})" - ) + # Only bites a caller-built tree: a parsed one derives total_bytes from + # the pool itself, and 0 means the caller knows no system total. + if memory.total_bytes and memory.memory_pool_bytes > memory.total_bytes: + self.errors.append("Hardware inventory: Spawn pool extends beyond total memory") - pool_end = memory.memory_pool_base + memory.memory_pool_bytes - if pool_end > iomem_end: - exceeds_by = pool_end - iomem_end - self.errors.append( - f"Hardware inventory: Memory pool extends beyond multikernel reserved pool in /proc/iomem " - f"by {exceeds_by} bytes ({exceeds_by / (1024**3):.2f} GB). " - f"Configured pool: {hex(memory.memory_pool_base)}-{hex(pool_end-1)}, " - f"Reserved pool: {hex(iomem_base)}-{hex(iomem_end-1)}" - ) + # A request names sizes only; the kernel picks the chunks, so there is + # nothing to line up with /proc/iomem until they are read back. + if memory.regions: + self._validate_pool_against_iomem(memory) - if memory.memory_pool_bytes > iomem_size: - self.errors.append( - f"Hardware inventory: Memory pool size ({memory.memory_pool_bytes} bytes = " - f"{memory.memory_pool_bytes / (1024**3):.2f} GB) exceeds multikernel reserved pool " - f"({iomem_size} bytes = {iomem_size / (1024**3):.2f} GB) in /proc/iomem" - ) - else: + def _validate_pool_against_iomem(self, memory): + """Every pool chunk the kernel reported must be registered in /proc/iomem.""" + iomem_chunks = set(get_pool_chunks_from_iomem()) + if not iomem_chunks: self.warnings.append("Could not find multikernel memory pool in /proc/iomem") + return + + missing = sorted({(r.base, r.size) for r in memory.regions} - iomem_chunks) + for base, size in missing: + self.errors.append( + f"Hardware inventory: Pool chunk {hex(base)}-{hex(base + size - 1)} " + f"({size / (1024**3):.2f} GB) is not registered in /proc/iomem" + ) def _validate_instances(self, tree: GlobalDeviceTree): """Validate instance definitions.""" @@ -383,34 +368,27 @@ def _validate_cpu_allocation(self, instance, tree: GlobalDeviceTree): def _validate_memory_allocation(self, instance, tree: GlobalDeviceTree): """Validate memory allocation for an instance.""" - memory = tree.hardware.memory instance_memory = instance.resources - memory_start = instance_memory.memory_base - memory_end = memory_start + instance_memory.memory_bytes - if memory_start < memory.memory_pool_base: - error_msg = self._format_error_with_context( - error_type="Memory allocation error", - instance_name=instance.name, - problem=f"Memory base {hex(memory_start)} is before memory pool start", - current_state=f"Spawn pool starts at {hex(memory.memory_pool_base)}", - suggestion=f"Use memory base >= {hex(memory.memory_pool_base)}", - alternative="Adjust memory pool configuration", - pattern=f"memory-base = <{hex(memory_start)}>", - ) - self.errors.append(error_msg) + # A zero base means the instance carries no placement of its own: the + # kernel picks one out of the pool when it creates the instance. + if not memory_start: + return - if memory_end > memory.memory_pool_base + memory.memory_pool_bytes: - pool_end = memory.memory_pool_base + memory.memory_pool_bytes - exceeds_by = memory_end - pool_end + memory_end = memory_start + instance_memory.memory_bytes + chunks = tree.hardware.memory.regions + + if chunks and chunk_containing(tree, memory_start, instance_memory.memory_bytes) is None: + listed = ", ".join(f"{hex(c.base)}-{hex(c.base + c.size - 1)}" for c in chunks) error_msg = self._format_error_with_context( error_type="Memory allocation error", instance_name=instance.name, - problem="Memory region extends beyond memory pool", - current_state=f"Instance memory: {hex(memory_start)}-{hex(memory_end)}, Pool: {hex(memory.memory_pool_base)}-{hex(pool_end)}", - suggestion=f"Reduce memory size by {hex(exceeds_by)} or use different base address", - alternative="Increase memory pool size or adjust memory allocation", + problem="Memory region does not fit in any pool chunk", + current_state=f"Instance memory: {hex(memory_start)}-{hex(memory_end)}, " + f"pool chunks: {listed}", + suggestion="Place the instance inside one chunk or reduce its memory size", + alternative="Grow the pool with 'kerf init --memory=...'", pattern=f"memory-size = <{hex(instance_memory.memory_bytes)}>", ) self.errors.append(error_msg) @@ -427,6 +405,8 @@ def _validate_memory_allocation(self, instance, tree: GlobalDeviceTree): continue other_start = other_instance.resources.memory_base + if not other_start: + continue other_end = other_start + other_instance.resources.memory_bytes if not (memory_end <= other_start or other_end <= memory_start): @@ -438,7 +418,8 @@ def _validate_memory_allocation(self, instance, tree: GlobalDeviceTree): f"Instance {instance.name} and {other_name}: Memory region overlap detected\n" f" {instance.name} memory: {hex(memory_start)} - {hex(memory_end)}\n" f" {other_name} memory: {hex(other_start)} - {hex(other_end)}\n" - f" Overlapping region: {hex(overlap_start)} - {hex(overlap_end)} ({overlap_size} bytes)" + f" Overlapping region: {hex(overlap_start)} - {hex(overlap_end)} " + f"({overlap_size} bytes)" ) def _validate_device_allocation(self, instance, tree: GlobalDeviceTree): diff --git a/src/kerf/init/main.py b/src/kerf/init/main.py index b3135ac..182c9d6 100644 --- a/src/kerf/init/main.py +++ b/src/kerf/init/main.py @@ -25,7 +25,7 @@ import re import sys from pathlib import Path -from typing import Optional +from typing import Dict, List, Optional, Tuple import click import libfdt @@ -38,8 +38,6 @@ from ..baseline import BaselineManager from ..create.main import parse_cpu_spec, parse_device_list, parse_memory_spec from ..dtc.parser import DeviceTreeParser -from ..lazy_cma import LAZY_CMA_DEVICE, allocate_multikernel_pool -from ..resources import get_memory_pool_from_iomem from ..dtc.reporter import ValidationReporter from ..dtc.validator import MultikernelValidator from ..exceptions import KernelInterfaceError, ParseError, ValidationError @@ -49,7 +47,12 @@ GlobalDeviceTree, HardwareInventory, MemoryAllocation, + PoolMemoryRegion, ) +from ..pool_diff import ANY_NODE, PoolDiff, compute_pool_diff +from ..resources import get_busy_chunks_from_iomem +from ..runtime import DeviceTreeManager +from ..topology import cpu_numa_nodes, node_for_cpus MULTIKERNEL_MOUNT_POINT = "/sys/fs/multikernel" @@ -353,34 +356,257 @@ def get_valid_apic_ids_from_system() -> Optional[set]: return None +_NODE_SPEC = re.compile(r"^(.+)@(.*)$") + +PAGE_SIZE = 4096 + + +NO_RESOURCE = "none" + + +def spec_is_empty(spec: str, option: str, synonyms: Tuple[str, ...] = ()) -> bool: + """ + Whether an option spells "none of this resource". + + Args: + spec: The option's value as typed on the command line + option: The option's name, for the error message + synonyms: Extra spellings that mean the same as "none" + + Returns: + True if the option asks for nothing + + Raises: + ValueError: If an empty spelling is mixed with real entries + """ + empty = {NO_RESOURCE} | set(synonyms) + parts = [p.strip().lower() for p in spec.split(",") if p.strip()] + asked = [p for p in parts if p in empty] + if not asked: + return False + if len(parts) > 1: + raise ValueError(f"{option}={asked[0]} cannot be combined with other entries") + return True + + +def parse_cpu_request(spec: str) -> List[int]: + """ + Parse the pool CPU request, where "none" asks for no pool CPUs. + + Args: + spec: APIC ID specification, or "none" + + Returns: + The requested APIC IDs, empty for "none" + + Raises: + ValueError: If the specification is malformed + """ + if spec_is_empty(spec, "--cpus"): + return [] + try: + return parse_cpu_spec(spec) + except ValueError as e: + raise ValueError(f"Invalid CPU specification '{spec}': {e}") from e + + +def parse_device_request(spec: Optional[str]) -> List[str]: + """ + Parse the pool device request, where "none" asks for no devices. + + Args: + spec: Comma-separated device names, "none", or None + + Returns: + The requested device names, empty for "none" and for no request + + Raises: + ValueError: If "none" is mixed with device names + """ + if not spec or spec_is_empty(spec, "--devices"): + return [] + return parse_device_list(spec) + + +def validate_memory_request(requested: Dict[int, int]) -> None: + """ + Reject pool sizes the kernel cannot honour, however they were asked for. + + Args: + requested: Mapping of NUMA node id (ANY_NODE if not resolved yet) to size + + Raises: + ValueError: If a size is zero, negative or not page aligned + """ + for node, size in sorted(requested.items()): + where = "the unpinned request" if node == ANY_NODE else f"node {node}" + if size <= 0: + raise ValueError(f"memory size for {where} must be greater than zero") + if size % PAGE_SIZE: + raise ValueError( + f"memory size for {where} must be a multiple of {PAGE_SIZE} bytes" + ) + + +def parse_memory_request(spec: str) -> Dict[int, int]: + """ + Parse a pool memory request into per-NUMA-node sizes. + + "2GB" asks for 2GB without naming a node, which the caller then + resolves to the node of the requested CPUs; "8GB@0,8GB@1" asks for a + specific amount per node, mirroring the device-tree unit address + convention. The two forms cannot be mixed. "none", or its synonym + "0", asks for no pool memory at all. + + Args: + spec: Memory specification string + + Returns: + Mapping of NUMA node id (-1 when the node is left to kerf) to size + in bytes, empty for "none" + + Raises: + ValueError: If the specification is malformed or a size is zero + or not page aligned + """ + if spec_is_empty(spec, "--memory", synonyms=("0",)): + return {} + + parts = [p.strip() for p in spec.split(",") if p.strip()] + if not parts: + raise ValueError("empty memory specification") + + requested: Dict[int, int] = {} + for part in parts: + match = _NODE_SPEC.match(part) + if match: + size_part, node_part = match.group(1), match.group(2) + try: + node = int(node_part) + except ValueError as exc: + raise ValueError( + f"invalid NUMA node '{node_part}' in '{part}' (expected SIZE@N)" + ) from exc + if node < 0: + raise ValueError(f"NUMA node in '{part}' must not be negative") + size = parse_memory_spec(size_part) + else: + node, size = ANY_NODE, parse_memory_spec(part) + if node in requested: + raise ValueError(f"node {node} specified twice") + requested[node] = size + + if ANY_NODE in requested and len(requested) > 1: + raise ValueError("cannot mix a plain size with SIZE@N entries") + validate_memory_request(requested) + return requested + + +def _cpu_ranges(cpus: List[int]) -> str: + """Render APIC ids the way they are written on the command line.""" + parts = [] + for cpu in sorted(set(cpus)): + if parts and cpu == parts[-1][1] + 1: + parts[-1][1] = cpu + else: + parts.append([cpu, cpu]) + return ",".join(str(a) if a == b else f"{a}-{b}" for a, b in parts) + + +def pick_memory_node(cpu_list: List[int], + pool_cpus: Optional[set] = None, + pool_regions: Optional[List[PoolMemoryRegion]] = None + ) -> Tuple[int, str]: + """ + Choose the NUMA node for a memory request that did not name one. + + Placement is policy, so kerf decides it and the kernel is only ever + handed an explicit node. + + Args: + cpu_list: APIC IDs the pool is being asked for + pool_cpus: APIC IDs the pool already holds + pool_regions: Chunks the pool already holds + + Returns: + The chosen node and a short reason to show the user + """ + node = node_for_cpus(cpu_list, cpu_numa_nodes()) + if node is not None: + return node, f"from CPUs {_cpu_ranges(cpu_list)}" + + # A CPU the pool already took is offline, so neither sysfs nor + # /proc/cpuinfo places it any more; its chunks still do. + if set(cpu_list) & set(pool_cpus or ()): + nodes = {r.node for r in (pool_regions or []) if r.node != ANY_NODE} + if len(nodes) == 1: + return nodes.pop(), "from the chunks the pool already holds" + + return 0, "no NUMA topology available, defaulting to node 0" + + +def resolve_memory_nodes(requested: Dict[int, int], + cpu_list: List[int], + pool_cpus: Optional[set] = None, + pool_regions: Optional[List[PoolMemoryRegion]] = None + ) -> Tuple[Dict[int, int], Optional[str]]: + """ + Pin an unpinned memory request to a node before the kernel sees it. + + Args: + requested: Mapping of NUMA node id (ANY_NODE for unpinned) to size + cpu_list: APIC IDs the pool is being asked for + pool_cpus: APIC IDs the pool already holds + pool_regions: Chunks the pool already holds + + Returns: + The request with every size on an explicit node, and a line + describing the choice, or None if nothing had to be resolved + """ + resolved = dict(requested) + size = resolved.pop(ANY_NODE, None) + if size is None: + return resolved, None + + node, why = pick_memory_node(cpu_list, pool_cpus, pool_regions) + resolved[node] = resolved.get(node, 0) + size + return resolved, f"Memory: {size >> 20} MB on node {node} ({why})" + + def build_baseline_from_cmdline( cpus: str, memory: Optional[str] = None, devices: Optional[str] = None, - verbose: bool = False + verbose: bool = False, + pool_cpus: Optional[set] = None, + pool_regions: Optional[List[PoolMemoryRegion]] = None ) -> GlobalDeviceTree: """ Build a GlobalDeviceTree from command line arguments. Args: - cpus: CPU specification string (e.g., "4-7" or "4,5,6,7") - memory: Optional pool size to allocate at runtime via /dev/lazy_cma - (e.g., "1GB"); when omitted an existing pool must already - be registered in /proc/iomem - devices: Optional device names (comma-separated, e.g., "enp9s0_dev,nvme0") + cpus: CPU specification string (e.g., "4-7" or "4,5,6,7"), or "none" + memory: Pool memory request, either "2GB" on the node of the + requested CPUs, "8GB@0,8GB@1" for specific nodes, or "none" + devices: Optional device names (comma-separated, e.g., "enp9s0_dev,nvme0"), + or "none" verbose: Whether to print verbose output + pool_cpus: APIC IDs the pool already holds + pool_regions: Chunks the pool already holds Returns: GlobalDeviceTree with resources only (no instances) Raises: - ValueError: If CPU or memory specification is invalid - KernelInterfaceError: If the memory pool cannot be found or allocated + ValueError: If the CPU or memory specification is invalid + KernelInterfaceError: If the system topology cannot be read """ - try: - cpu_list = parse_cpu_spec(cpus) - except ValueError as e: - raise ValueError(f"Invalid CPU specification '{cpus}': {e}") from e + cpu_list = parse_cpu_request(cpus) + + # Silence about memory would otherwise read as "give it all back". + if not memory: + raise ValueError("--memory is required") + requested = parse_memory_request(memory) # Validate against valid APIC IDs on the system valid_apic_ids = get_valid_apic_ids_from_system() @@ -390,6 +616,10 @@ def build_baseline_from_cmdline( "Ensure the system exposes CPU topology information." ) + # The host stops listing a CPU in /proc/cpuinfo once the pool takes it, + # so a re-init has to count the pool's own CPUs as valid. + valid_apic_ids = valid_apic_ids | set(pool_cpus or ()) + invalid_cpus = set(cpu_list) - valid_apic_ids if invalid_cpus: raise ValueError( @@ -410,45 +640,19 @@ def build_baseline_from_cmdline( host_reserved_cpus = [0] cpu_list = sorted(list(available_cpus)) - memory_pool = get_memory_pool_from_iomem() - if memory_pool is None: - if not memory: - raise KernelInterfaceError( - "Could not find multikernel memory pool in /proc/iomem. " - "Pass --memory=SIZE (e.g. --memory=1GB) to allocate the pool " - f"at runtime via {LAZY_CMA_DEVICE}." - ) - - pool_bytes = parse_memory_spec(memory) - pool_base = allocate_multikernel_pool(pool_bytes) - if verbose: - click.echo(f"Allocated multikernel memory pool via {LAZY_CMA_DEVICE}:") - click.echo(f" Base: {hex(pool_base)}") - click.echo(f" Size: {pool_bytes} bytes ({pool_bytes / (1024**3):.2f} GB)") - - # Re-read /proc/iomem so the baseline reflects exactly what the - # kernel registered for the allocation. - memory_pool = get_memory_pool_from_iomem() or (pool_base, pool_bytes) - elif memory: - click.echo( - "Note: multikernel memory pool already exists in /proc/iomem; " - "ignoring --memory", - err=True, - ) - memory_pool_base, memory_pool_bytes = memory_pool + requested, note = resolve_memory_nodes(requested, cpu_list, pool_cpus, pool_regions) + if note: + click.echo(note) - total_bytes = memory_pool_base + memory_pool_bytes - host_reserved_bytes = memory_pool_base + total_bytes = sum(requested.values()) if verbose: click.echo(f"Parsed APIC ID specification: {cpus}") click.echo(f" Valid APIC IDs on system: {sorted(valid_apic_ids)}") click.echo(f" Host-reserved APIC IDs: {host_reserved_cpus}") click.echo(f" Available APIC IDs: {cpu_list}") - click.echo("Memory pool from /proc/iomem:") - click.echo(f" Base: {hex(memory_pool_base)}") - click.echo(f" Size: {memory_pool_bytes} bytes ({memory_pool_bytes / (1024**3):.2f} GB)") - click.echo(f" Total bytes: {total_bytes} bytes ({total_bytes / (1024**3):.2f} GB)") - click.echo(f" Host-reserved: {host_reserved_bytes} bytes ({host_reserved_bytes / (1024**3):.2f} GB)") + click.echo("Requested pool memory:") + for node, size in sorted(requested.items()): + click.echo(f" node {node}: {size} bytes ({size / (1024**3):.2f} GB)") cpu_allocation = CPUAllocation( total=total_cpus, @@ -458,14 +662,13 @@ def build_baseline_from_cmdline( memory_allocation = MemoryAllocation( total_bytes=total_bytes, - host_reserved_bytes=host_reserved_bytes, - memory_pool_base=memory_pool_base, - memory_pool_bytes=memory_pool_bytes + host_reserved_bytes=0, + requested=requested ) device_dict = {} - if devices: - device_names = parse_device_list(devices) + device_names = parse_device_request(devices) + if device_names: for device_name in device_names: device_info = detect_device_from_system(device_name) if device_info: @@ -503,14 +706,216 @@ def build_baseline_from_cmdline( return tree +INSTANCES_DIR = "/sys/fs/multikernel/instances" + + +def list_instance_names(instances_dir: str = INSTANCES_DIR) -> List[str]: + """Names of the instances the kernel currently holds.""" + path = Path(instances_dir) + if not path.exists(): + return [] + try: + return sorted(child.name for child in path.iterdir() if child.is_dir()) + except OSError: + return [] + + +def pool_is_live(current: Optional[GlobalDeviceTree]) -> bool: + """ + Whether the kernel already holds pool resources we have to diff against. + + A non-empty cpus list is not evidence: before a pool exists the root + read-back lists the host's own online CPUs there. Only the pool branch + of the kernel's device tree emits cpus-available, memory@N or devices. + """ + if current is None: + return False + hardware = current.hardware + return bool(hardware.cpus.available_free is not None + or hardware.memory.regions + or hardware.devices) + + +def pool_apic_ids(current: Optional[GlobalDeviceTree]) -> set: + """APIC IDs the live pool holds, which the host no longer reports.""" + if not pool_is_live(current): + return set() + return set(current.hardware.cpus.available or []) + + +def pool_memory_regions(current: Optional[GlobalDeviceTree]) -> List[PoolMemoryRegion]: + """Chunks the live pool holds, which place CPUs the host no longer reports.""" + if not pool_is_live(current): + return [] + return list(current.hardware.memory.regions) + + +def read_current_pool(baseline_mgr) -> Optional[GlobalDeviceTree]: + """ + Read the live pool back from the kernel. + + Before any pool exists the kernel still publishes a root device tree, + but one that describes the host rather than a pool and that carries no + memory node, so a failed read means "no pool yet", not an error. + + Args: + baseline_mgr: BaselineManager to read through + + Returns: + The baseline the kernel reports, or None if there is no pool yet + """ + try: + return baseline_mgr.read_baseline() + except (ParseError, KernelInterfaceError): + return None + + +def request_is_empty(requested: GlobalDeviceTree) -> bool: + """Whether the request asks for nothing at all, so the pool goes away.""" + hardware = requested.hardware + return not (hardware.cpus.available or hardware.memory.requested or hardware.devices) + + +def _print_diff(diff: PoolDiff) -> None: + """Show what would move between the host and the pool.""" + def line(label, items): + if items: + click.echo(f" {label}: {', '.join(str(i) for i in items)}") + + line("CPUs to pool", diff.cpus_to_pool) + line("CPUs to host", diff.cpus_to_host) + line("Memory to pool", [ + f"{size >> 20} MB" + ("" if node == ANY_NODE else f" on node {node}") + for node, size in diff.memory_to_pool + ]) + line("Memory to host", [f"{hex(r.base)} ({r.size >> 20} MB)" for r in diff.memory_to_host]) + line("Devices to pool", diff.devices_to_pool) + line("Devices to host", diff.devices_to_host) + + +def _report_shortfall(live: GlobalDeviceTree, requested: GlobalDeviceTree) -> None: + """Warn when the kernel could not shrink the pool all the way down.""" + for node, want in requested.hardware.memory.requested.items(): + if node == ANY_NODE: + have = live.hardware.memory.memory_pool_bytes + else: + have = live.hardware.memory.bytes_on_node(node) + if have > want: + where = "any" if node == ANY_NODE else node + click.echo( + f"Note: node {where} still holds {(have - want) >> 20} MB more than " + "requested; only whole idle chunks can be returned", + err=True, + ) + + +def reconcile_pool( + current: Optional[GlobalDeviceTree], + requested: GlobalDeviceTree, + busy_chunks: set, + dry_run: bool, + manager, + baseline_mgr, +) -> Optional[PoolDiff]: + """ + Bring the live pool in line with the requested baseline. + + An empty pool takes the baseline write; a live pool is reconciled with + a /resources overlay transaction, since the kernel refuses a baseline + write once it owns resources. + + Args: + current: Baseline read back from the kernel, or None if there is none + requested: Requested state + busy_chunks: Bases of pool chunks that still hold an allocation + dry_run: Report the plan without touching the kernel + manager: DeviceTreeManager used to apply the overlay + baseline_mgr: BaselineManager used for the initial write + + Returns: + The applied (or planned) difference, or None if a baseline was written + + Raises: + KernelInterfaceError: If the kernel rejects the write or the overlay + ValidationError: If emptying the pool would strand running instances + """ + if not pool_is_live(current): + # The kernel rejects a baseline with no memory@N node, and there is + # nothing to hand back anyway. + if request_is_empty(requested): + click.echo("Pool is already empty; nothing to do") + return None + if dry_run: + click.echo("Baseline validation passed; would write the initial baseline (dry-run)") + return None + baseline_mgr.write_baseline(requested) + click.echo("✓ Baseline applied to kernel successfully") + return None + + diff = compute_pool_diff(current, requested, busy_chunks=busy_chunks) + _print_diff(diff) + if diff.is_empty(): + click.echo("✓ Pool already matches the request; nothing to do") + _report_shortfall(current, requested) + return diff + if dry_run: + click.echo("Would apply the changes above (dry-run)") + _report_shortfall(current, requested) + return diff + + if request_is_empty(requested): + held = list_instance_names() + if held: + raise ValidationError( + f"The pool still runs {len(held)} instance(s): " + f"delete instances {', '.join(held)} first" + ) + + with manager.lock(): + tx_id = manager.apply_dtbo(manager.overlay_gen.generate_pool_overlay(diff)) + click.echo(f"✓ Pool updated (transaction {tx_id})") + + if request_is_empty(requested): + # /resources carries no memory@N once the pool is gone, so there is + # nothing left to read back or compare against. + return diff + + try: + live = baseline_mgr.read_baseline() + except (ParseError, KernelInterfaceError) as e: + # The transaction already landed; a failed read-back is worth a word, + # not a failure. + click.echo(f"Note: could not read the pool back after the transaction: {e}", err=True) + else: + _report_shortfall(live, requested) + return diff + + +def _dump_baseline_dts(baseline_mgr: BaselineManager, tree: GlobalDeviceTree) -> None: + """Print the DTS the baseline write would carry.""" + try: + dtb_data = baseline_mgr.extractor.generate_global_dtb(tree) + fdt = libfdt.Fdt(dtb_data) + dts_parser = DeviceTreeParser() + dts_parser.fdt = fdt + dts_lines = dts_parser._fdt_to_dts_recursive(0, 0) # pylint: disable=protected-access + + click.echo("Debug: Baseline DTS source being written to kernel:") + click.echo("─" * 70) + click.echo('\n'.join(dts_lines)) + click.echo("─" * 70) + except Exception as e: # pylint: disable=broad-except + click.echo(f"Debug: Failed to convert baseline DTB to DTS: {e}", err=True) + + @click.command() @click.pass_context @click.option('--input', '-i', help='Input DTS or DTB file containing all resources. Mutually exclusive with --cpus, --memory and --devices. When used, all resources must come from the file.') -@click.option('--cpus', '-c', help='APIC ID specification for baseline (e.g., "128-134" or "128,130,132"). Use physical APIC IDs, not logical CPU numbers. Mutually exclusive with --input. Memory comes from --memory or an existing pool in /proc/iomem.') -@click.option('--memory', '-m', help='Memory pool size to allocate at runtime via /dev/lazy_cma (e.g., "1GB", "512MB"). If omitted, an existing pool is discovered from /proc/iomem. Mutually exclusive with --input.') -@click.option('--devices', '-d', help='Device names (comma-separated, e.g., "enp9s0_dev,nvme0"). Mutually exclusive with --input. Creates minimal device entries in baseline.') -@click.option('--dry-run', is_flag=True, help='Validate without applying') -@click.option('--report', is_flag=True, help='Generate detailed validation report') +@click.option('--cpus', '-c', help='APIC ID specification for baseline (e.g., "128-134" or "128,130,132"), or "none" for no pool CPUs. Use physical APIC IDs, not logical CPU numbers. Mutually exclusive with --input.') +@click.option('--memory', '-m', help='Pool memory: SIZE (e.g. "2GB") on the node of the requested CPUs, per-node "8GB@0,8GB@1", or "none" for no pool memory. Required with --cpus, mutually exclusive with --input.') +@click.option('--devices', '-d', help='Device names (comma-separated, e.g., "enp9s0_dev,nvme0"), or "none" for no devices. Mutually exclusive with --input. Creates minimal device entries in baseline.') +@click.option('--dry-run', is_flag=True, help='Report the plan without applying it. Still reads the pool from the kernel, so it needs root.') +@click.option('--report', is_flag=True, help='Generate detailed validation report. Ignored when the request asks for no memory.') @click.option('--format', type=click.Choice(['text', 'json', 'yaml']), default='text', help='Report format (default: text)') @click.option('--verbose', '-v', is_flag=True, help='Verbose output') @@ -522,31 +927,46 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: available for allocation. The baseline must contain ONLY resources (no instances). Instances are created via 'kerf create' using overlays. - By default, the baseline is applied to the kernel after validation. - Use --dry-run to validate without applying. + The command is idempotent: the requested state is compared against the + pool the kernel reports, and only the difference is applied. An empty + pool takes the baseline write; a live pool is reconciled with a + /resources overlay transaction. Use --dry-run to see the plan without + applying it. Even --dry-run reads the pool from the kernel, so every + form needs root. You can either provide a DTS/DTB file via --input, or construct the - baseline from command line arguments using --cpus. These options are - mutually exclusive - when using --input, all resources must come from - the DTS file. With --cpus, the memory pool is allocated at runtime - via /dev/lazy_cma when --memory is given, or discovered from - /proc/iomem otherwise. + request from command line arguments using --cpus and --memory. These + options are mutually exclusive. + + Every resource is spelled out: --cpus=none, --memory=none (or + --memory=0) and --devices=none ask for none of that resource, and a + request that asks for nothing returns the whole pool to the host. + + A --memory size that names no node is placed on the NUMA node of the + requested CPUs. Kerf resolves it here so the kernel is always handed an + explicit node and never picks the placement itself. Examples: # Initialize from DTS file (all resources from file) kerf init --input=hardware.dts - # Initialize from command line, allocating a 1GB pool at runtime + # Request 1GB of pool memory on the node of the requested CPUs kerf init --cpus=128-134 --memory=1GB - # Initialize reusing a pool already registered in /proc/iomem - kerf init --cpus=128-134 + # Request memory per NUMA node + kerf init --cpus=128-134 --memory=8GB@0,8GB@1 - # Initialize with APIC IDs and devices - kerf init --cpus=128,130,132 --memory=1GB --devices=enp9s0_dev,nvme0 + # Shrink the pool back to 1GB and 2 CPUs + kerf init --cpus=128,129 --memory=1GB - # Validate baseline without applying - kerf init --input=hardware.dts --dry-run + # Keep the CPUs but hand every chunk back + kerf init --cpus=128,129 --memory=none + + # Return every pool resource to the host + kerf init --cpus=none --memory=none + + # Show what would change without applying + kerf init --cpus=128-134 --memory=1GB --dry-run """ try: # Validate that --input and resource specification options are mutually exclusive @@ -569,12 +989,20 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: click.echo("\nUsage:", err=True) click.echo(" kerf init --input=hardware.dts", err=True) click.echo(" kerf init --cpus=4-7 --memory=1GB", err=True) - click.echo(" kerf init --cpus=4-7 --memory=1GB --devices=enp9s0_dev", err=True) + click.echo(" kerf init --cpus=4-7 --memory=1GB@0 --devices=enp9s0_dev", err=True) + click.echo(" kerf init --cpus=none --memory=none", err=True) sys.exit(2) parser = DeviceTreeParser() dts_content = None + baseline_mgr = BaselineManager() + manager = DeviceTreeManager() + mount_multikernel_fs(verbose=verbose) + current = read_current_pool(baseline_mgr) + live_cpus = pool_apic_ids(current) + live_regions = pool_memory_regions(current) + if input: # Parse from input file input_path = Path(input) @@ -593,10 +1021,24 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: click.echo(f"Error: Unsupported input format: {input_path.suffix}", err=True) click.echo("Supported formats: .dts, .dtb", err=True) sys.exit(2) + + try: + validate_memory_request(tree.hardware.memory.requested) + except ValueError as e: + click.echo(f"Error: {input}: {e}", err=True) + sys.exit(2) + + tree.hardware.memory.requested, note = resolve_memory_nodes( + tree.hardware.memory.requested, tree.hardware.cpus.available, + live_cpus, live_regions) + if note: + click.echo(note) else: # Build from command line arguments try: - tree = build_baseline_from_cmdline(cpus, memory=memory, devices=devices, verbose=verbose) + tree = build_baseline_from_cmdline(cpus, memory=memory, devices=devices, + verbose=verbose, pool_cpus=live_cpus, + pool_regions=live_regions) except ValueError as e: click.echo(f"Error: {e}", err=True) sys.exit(2) @@ -604,8 +1046,6 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: click.echo(f"Error: {e}", err=True) sys.exit(1) - baseline_mgr = BaselineManager() - try: baseline_mgr.validate_baseline(tree) except ValidationError as e: @@ -616,74 +1056,59 @@ def init(ctx: click.Context, input: Optional[str], cpus: Optional[str], memory: click.echo("\nInstances should be created via 'kerf create'", err=True) sys.exit(1) - validator = MultikernelValidator() - if dts_content is not None: - input_path_str = str(input) if input else "command-line" - validator.set_dts_context(dts_content, input_path_str) + # The resource validator reads a pool with no memory as unusable, + # which is exactly what a request for no memory asks for. + if tree.hardware.memory.requested: + validator = MultikernelValidator() + if dts_content is not None: + input_path_str = str(input) if input else "command-line" + validator.set_dts_context(dts_content, input_path_str) - validation_result = validator.validate(tree) + validation_result = validator.validate(tree) + + if report: + reporter = ValidationReporter() + report_text = reporter.generate_report(validation_result, tree, verbose, format) + click.echo(report_text) + if not validation_result.is_valid: + sys.exit(1) + return - if report: - reporter = ValidationReporter() - report_text = reporter.generate_report(validation_result, tree, verbose, format) - click.echo(report_text) if not validation_result.is_valid: + click.echo("Validation failed:", err=True) + for error in validation_result.errors: + click.echo(f" ✗ {error}", err=True) + if validation_result.warnings: + click.echo("\nWarnings:", err=True) + for warning in validation_result.warnings: + click.echo(f" ⚠ {warning}", err=True) sys.exit(1) - return - - if not validation_result.is_valid: - click.echo("Validation failed:", err=True) - for error in validation_result.errors: - click.echo(f" ✗ {error}", err=True) - if validation_result.warnings: - click.echo("\nWarnings:", err=True) - for warning in validation_result.warnings: - click.echo(f" ⚠ {warning}", err=True) - sys.exit(1) - if verbose: - click.echo("✓ Baseline validation passed") - if validation_result.warnings: - click.echo("\nWarnings:") - for warning in validation_result.warnings: - click.echo(f" ⚠ {warning}") + if verbose: + click.echo("✓ Baseline validation passed") + if validation_result.warnings: + click.echo("\nWarnings:") + for warning in validation_result.warnings: + click.echo(f" ⚠ {warning}") debug = ctx.obj.get('debug', False) if ctx and ctx.obj else False - if dry_run: - click.echo(" Baseline validation passed") - click.echo(" Baseline would be applied (dry-run mode)") - else: - try: - mount_multikernel_fs(verbose=verbose) - - if debug: - try: - dtb_data = baseline_mgr.extractor.generate_global_dtb(tree) - fdt = libfdt.Fdt(dtb_data) - dts_parser = DeviceTreeParser() - dts_parser.fdt = fdt - dts_lines = dts_parser._fdt_to_dts_recursive(0, 0) # pylint: disable=protected-access - dts_content = '\n'.join(dts_lines) - - click.echo("Debug: Baseline DTS source being written to kernel:") - click.echo("─" * 70) - click.echo(dts_content) - click.echo("─" * 70) - except Exception as e: - click.echo(f"Debug: Failed to convert baseline DTB to DTS: {e}", err=True) + if debug and not pool_is_live(current): + _dump_baseline_dts(baseline_mgr, tree) - if verbose: - click.echo("Writing baseline to kernel...") - baseline_mgr.write_baseline(tree) - click.echo("✓ Baseline applied to kernel successfully") - click.echo(" Baseline: /sys/fs/multikernel/device_tree") - except KernelInterfaceError as e: - click.echo(f"Error: Failed to apply baseline: {e}", err=True) - if verbose: - import traceback - traceback.print_exc() - sys.exit(1) + try: + reconcile_pool(current, tree, get_busy_chunks_from_iomem(), dry_run, + manager, baseline_mgr) + except ValidationError as e: + click.echo(f"Error: {e}", err=True) + sys.exit(1) + except KernelInterfaceError as e: + click.echo(f"Error: {e}", err=True) + click.echo("Run 'dmesg | tail' for the kernel's reason (it names the busy CPU or chunk).", err=True) + if verbose: + import traceback + traceback.print_exc() + sys.exit(1) except ParseError as e: click.echo(f"Error: Failed to parse input file: {e}", err=True) diff --git a/src/kerf/lazy_cma.py b/src/kerf/lazy_cma.py deleted file mode 100644 index b204433..0000000 --- a/src/kerf/lazy_cma.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright 2026 Multikernel Technologies, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Client for the lazy_cma runtime contiguous memory allocator. - -kerf allocates the multikernel memory pool at runtime through -/dev/lazy_cma instead of requiring a boot-time reservation. Allocating -under the canonical resource name ("Multikernel Memory Pool") makes the -pool discoverable in /proc/iomem, which is where kerf and the kernel -tooling look it up. -""" - -import fcntl -import os -import struct - -from .exceptions import KernelInterfaceError - -LAZY_CMA_DEVICE = "/dev/lazy_cma" - -#: iomem resource name the multikernel stack discovers the pool by. -MULTIKERNEL_POOL_NAME = "Multikernel Memory Pool" - -_LAZY_CMA_IOC_MAGIC = ord("H") -_LAZY_CMA_NAME_MAX = 64 - -# struct lazy_cma_allocation_data { -# __u64 len; -# __u64 phys_addr; /* out */ -# __s32 node; /* NUMA node; -1 = any */ -# __u32 pad; -# char name[64]; /* iomem resource name */ -# }; -_ALLOC_FORMAT = f" int: - """Encode an ioctl request number (matches the kernel's _IOC macro).""" - return (direction << 30) | (size << 16) | (_LAZY_CMA_IOC_MAGIC << 8) | nr - - -LAZY_CMA_IOCTL_ALLOC = _ioc(_IOC_READ | _IOC_WRITE, 0x0, _ALLOC_SIZE) - - -def allocate(size_bytes: int, name: str, node: int = -1) -> int: - """ - Allocate physically contiguous memory via /dev/lazy_cma. - - Args: - size_bytes: Allocation size in bytes (page granularity in the kernel) - name: iomem resource name for the allocation - node: NUMA node to allocate from, or -1 for any node - - Returns: - Physical base address of the allocation - - Raises: - KernelInterfaceError: If the device is unavailable or allocation fails - ValueError: If the arguments are invalid - """ - if size_bytes <= 0: - raise ValueError(f"Allocation size must be positive, got {size_bytes}") - - encoded_name = name.encode("utf-8") - if len(encoded_name) >= _LAZY_CMA_NAME_MAX: - raise ValueError( - f"Resource name too long ({len(encoded_name)} bytes, " - f"max {_LAZY_CMA_NAME_MAX - 1})" - ) - - request = bytearray( - struct.pack(_ALLOC_FORMAT, size_bytes, 0, node, 0, encoded_name) - ) - - try: - fd = os.open(LAZY_CMA_DEVICE, os.O_RDWR) - except FileNotFoundError as exc: - raise KernelInterfaceError( - f"{LAZY_CMA_DEVICE} not found. Load the lazy_cma kernel module " - "first (e.g. insmod lazy_cma.ko)." - ) from exc - except PermissionError as exc: - raise KernelInterfaceError( - f"Permission denied opening {LAZY_CMA_DEVICE}. kerf must run as " - "root to allocate pool memory." - ) from exc - - try: - fcntl.ioctl(fd, LAZY_CMA_IOCTL_ALLOC, request) - except OSError as exc: - raise KernelInterfaceError( - f"lazy_cma allocation of {size_bytes} bytes failed: " - f"{os.strerror(exc.errno)} (errno {exc.errno}). " - "The system may not have enough contiguous free memory; " - "try a smaller pool size." - ) from exc - finally: - os.close(fd) - - _, phys_addr, _, _, _ = struct.unpack(_ALLOC_FORMAT, bytes(request)) - if phys_addr == 0: - raise KernelInterfaceError( - f"lazy_cma returned physical address 0 for a {size_bytes} byte " - "allocation" - ) - - return phys_addr - - -def allocate_multikernel_pool(size_bytes: int, node: int = -1) -> int: - """ - Allocate the multikernel memory pool under its canonical iomem name. - - Returns the physical base address of the pool. - """ - return allocate(size_bytes, MULTIKERNEL_POOL_NAME, node) diff --git a/src/kerf/models.py b/src/kerf/models.py index 35667a3..c3fb3b7 100644 --- a/src/kerf/models.py +++ b/src/kerf/models.py @@ -16,7 +16,7 @@ Data models for multikernel device tree representation. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import List, Dict, Optional, Set, Tuple from enum import Enum @@ -93,28 +93,54 @@ class CPUAllocation: total: int host_reserved: List[int] - available: List[int] + available: List[int] # Pool membership: every CPU in the pool, lent or free topology: Optional[Dict[int, CPUTopology]] = None # CPU ID -> topology info + available_free: Optional[List[int]] = None # Pool members not lent to an instance def get_allocated_cpus(self) -> Set[int]: """Get set of CPUs allocated to instances.""" return set(self.available) - set(self.host_reserved) +@dataclass(frozen=True) +class PoolMemoryRegion: + """One kernel-owned pool chunk as reported by /sys/fs/multikernel/device_tree.""" + + base: int + size: int + node: int = -1 + + @dataclass class MemoryAllocation: - """Memory allocation information.""" + """Pool memory: live chunks from the kernel and per-node sizes requested by the user.""" total_bytes: int host_reserved_bytes: int - memory_pool_base: int - memory_pool_bytes: int + regions: List[PoolMemoryRegion] = field(default_factory=list) + requested: Dict[int, int] = field(default_factory=dict) + + @property + def memory_pool_base(self) -> int: + """Base address of the first live pool chunk, or 0 before any chunks are read back.""" + return self.regions[0].base if self.regions else 0 + + @property + def memory_pool_bytes(self) -> int: + """Total pool size: sum of live chunks, or sum of requested sizes if none yet.""" + if self.regions: + return sum(r.size for r in self.regions) + return sum(self.requested.values()) @property def memory_pool_end(self) -> int: """End address of memory pool.""" return self.memory_pool_base + self.memory_pool_bytes + def bytes_on_node(self, node: int) -> int: + """Total live pool bytes on the given NUMA node.""" + return sum(r.size for r in self.regions if r.node == node) + @dataclass class DeviceInfo: diff --git a/src/kerf/pool_diff.py b/src/kerf/pool_diff.py new file mode 100644 index 0000000..abab1cd --- /dev/null +++ b/src/kerf/pool_diff.py @@ -0,0 +1,98 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Difference between the live resource pool and a requested baseline.""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Set, Tuple + +from .models import GlobalDeviceTree, PoolMemoryRegion + +ANY_NODE = -1 + + +@dataclass +class PoolDiff: + cpus_to_pool: List[int] = field(default_factory=list) + cpus_to_host: List[int] = field(default_factory=list) + devices_to_pool: List[str] = field(default_factory=list) + devices_to_host: List[str] = field(default_factory=list) + memory_to_pool: List[Tuple[int, int]] = field(default_factory=list) + memory_to_host: List[PoolMemoryRegion] = field(default_factory=list) + + def is_empty(self) -> bool: + return not any((self.cpus_to_pool, self.cpus_to_host, self.devices_to_pool, + self.devices_to_host, self.memory_to_pool, self.memory_to_host)) + + +def _pci_ids(tree: GlobalDeviceTree) -> Set[str]: + return {d.pci_id for d in tree.hardware.devices.values() if d.pci_id} + + +def _memory_diff(regions: List[PoolMemoryRegion], requested: Dict[int, int], + busy: Set[int], diff: PoolDiff) -> None: + remaining = list(regions) + + for node, want in requested.items(): + if node == ANY_NODE: + continue + have = sum(r.size for r in remaining if r.node == node) + if want > have: + diff.memory_to_pool.append((node, want - have)) + elif want < have: + _release(remaining, lambda r, node=node: r.node == node, have - want, busy, diff) + + # Chunks on nodes nobody asked for are surplus, unless an any-node request absorbs them. + explicit = {n for n in requested if n != ANY_NODE} + any_want = requested.get(ANY_NODE, 0) + unclaimed = [r for r in remaining if r.node not in explicit] + have = sum(r.size for r in unclaimed) + if any_want > have: + diff.memory_to_pool.append((ANY_NODE, any_want - have)) + elif any_want < have: + _release(remaining, lambda r: r.node not in explicit, have - any_want, busy, diff) + + +def _release(remaining: List[PoolMemoryRegion], pred, surplus: int, + busy: Set[int], diff: PoolDiff) -> None: + # A chunk that still holds an allocation cannot go back to the host, and + # asking anyway fails the whole transaction. + candidates = sorted((r for r in remaining if pred(r) and r.base not in busy), + key=lambda r: -r.size) + for r in candidates: + if surplus <= 0: + break + if r.size > surplus: + continue + diff.memory_to_host.append(r) + remaining.remove(r) + surplus -= r.size + + +def compute_pool_diff(current: GlobalDeviceTree, requested: GlobalDeviceTree, + busy_chunks: Optional[Set[int]] = None) -> PoolDiff: + """The request is the desired state; memory on nodes it omits counts as surplus.""" + diff = PoolDiff() + cur_cpus = set(current.hardware.cpus.available) + req_cpus = set(requested.hardware.cpus.available) + diff.cpus_to_pool = sorted(req_cpus - cur_cpus) + diff.cpus_to_host = sorted(cur_cpus - req_cpus) + + cur_dev, req_dev = _pci_ids(current), _pci_ids(requested) + diff.devices_to_pool = sorted(req_dev - cur_dev) + diff.devices_to_host = sorted(cur_dev - req_dev) + + _memory_diff(current.hardware.memory.regions, requested.hardware.memory.requested, + busy_chunks or set(), diff) + return diff diff --git a/src/kerf/resources.py b/src/kerf/resources.py index 8cb7461..43cbdd6 100644 --- a/src/kerf/resources.py +++ b/src/kerf/resources.py @@ -24,12 +24,14 @@ from pathlib import Path from typing import List, Set, Optional, Tuple -from .lazy_cma import MULTIKERNEL_POOL_NAME -from .models import GlobalDeviceTree +from .models import GlobalDeviceTree, PoolMemoryRegion from .exceptions import ResourceError IOMEM_PATH = "/proc/iomem" +#: iomem resource name the kernel registers every pool chunk under. +MULTIKERNEL_POOL_NAME = "Multikernel Memory Pool" + _IOMEM_RANGE_RE = re.compile(r"([0-9a-fA-F]+)-([0-9a-fA-F]+)\s*:\s*(.*)") @@ -52,47 +54,84 @@ def _parse_iomem_regions(iomem_path: str) -> List[Tuple[int, int, str]]: return regions +def _pool_chunks(regions: List[Tuple[int, int, str]]) -> List[Tuple[int, int]]: + """Pick the pool chunks out of already parsed /proc/iomem regions.""" + return [ + (base, end - base + 1) + for base, end, name in regions + if MULTIKERNEL_POOL_NAME in name + ] + + +def _chunk_children(regions: List[Tuple[int, int, str]], + chunks: List[Tuple[int, int]]) -> List[Tuple[int, int, int]]: + """Map every region nested in a pool chunk to (chunk_base, base, end).""" + children = [] + for base, end, name in regions: + if MULTIKERNEL_POOL_NAME in name: + continue + for chunk_base, chunk_size in chunks: + if chunk_base <= base and end <= chunk_base + chunk_size - 1: + children.append((chunk_base, base, end)) + break + return children + + +def get_pool_chunks_from_iomem(iomem_path: str = IOMEM_PATH) -> List[Tuple[int, int]]: + """ + List every multikernel pool chunk registered in /proc/iomem. + + Returns: + (base_address, size_bytes) tuples in /proc/iomem order + """ + return _pool_chunks(_parse_iomem_regions(iomem_path)) + + +def get_busy_chunks_from_iomem(iomem_path: str = IOMEM_PATH) -> Set[int]: + """ + Find the pool chunks that still hold an allocation. + + A chunk with any nested region cannot be returned to the host, so the + pool diff avoids picking it when it has a choice. + + Returns: + Base addresses of the chunks with at least one child region + """ + regions = _parse_iomem_regions(iomem_path) + return {chunk_base for chunk_base, _, _ in _chunk_children(regions, _pool_chunks(regions))} + + def get_memory_pool_from_iomem(iomem_path: str = IOMEM_PATH) -> Optional[Tuple[int, int]]: """ - Get the multikernel memory pool region from /proc/iomem. + Get the first multikernel memory pool chunk from /proc/iomem. Returns: (base_address, size_bytes) or None if the pool is not registered """ - for base, end, name in _parse_iomem_regions(iomem_path): - if MULTIKERNEL_POOL_NAME in name: - return (base, end - base + 1) - return None + chunks = get_pool_chunks_from_iomem(iomem_path) + return chunks[0] if chunks else None def get_pool_allocated_bytes(iomem_path: str = IOMEM_PATH) -> Optional[Tuple[int, int, int]]: """ Compute pool usage from /proc/iomem, the single source of truth. - Every region nested inside the pool range (instance memory, daxfs + Every region nested inside a pool chunk (instance memory, daxfs heaps, ...) counts as allocated. Overlapping and nested child regions are merged so nothing is double counted. Returns: - (pool_base, pool_bytes, allocated_bytes), or None if the pool is - not registered in /proc/iomem + (first_chunk_base, pool_bytes, allocated_bytes), or None if the + pool is not registered in /proc/iomem """ - pool = get_memory_pool_from_iomem(iomem_path) - if pool is None: + regions = _parse_iomem_regions(iomem_path) + chunks = _pool_chunks(regions) + if not chunks: return None - pool_base, pool_bytes = pool - pool_end = pool_base + pool_bytes - 1 - - children = [] - for base, end, name in _parse_iomem_regions(iomem_path): - if MULTIKERNEL_POOL_NAME in name: - continue - if base >= pool_base and end <= pool_end: - children.append((base, end)) allocated = 0 current_base = current_end = None - for base, end in sorted(children): + for _, base, end in sorted(_chunk_children(regions, chunks), key=lambda c: c[1:]): if current_base is None: current_base, current_end = base, end elif base <= current_end + 1: @@ -103,28 +142,30 @@ def get_pool_allocated_bytes(iomem_path: str = IOMEM_PATH) -> Optional[Tuple[int if current_base is not None: allocated += current_end - current_base + 1 - return (pool_base, pool_bytes, allocated) + return (chunks[0][0], sum(size for _, size in chunks), allocated) def get_available_cpus(tree: GlobalDeviceTree) -> Set[int]: """ Get set of CPUs available for allocation (not allocated to any instance). + The kernel's own free list wins when the tree carries one: the root + read-back has no instances section, so deriving free CPUs from pool + membership alone would hand out CPUs that are already lent out. + Args: tree: GlobalDeviceTree to analyze Returns: Set of available CPU IDs """ - # Get all CPUs in the available pool - available = set(tree.hardware.cpus.available) + allocated = get_allocated_cpus(tree) - # Subtract CPUs allocated to instances - allocated = set() - for instance in tree.instances.values(): - allocated.update(instance.resources.cpus) + free = tree.hardware.cpus.available_free + if free is not None: + return set(free) - allocated - return available - allocated + return set(tree.hardware.cpus.available) - allocated def get_allocated_cpus(tree: GlobalDeviceTree) -> Set[int]: @@ -183,63 +224,23 @@ def get_allocated_memory_regions(tree: GlobalDeviceTree) -> List[tuple[int, int] return regions -def find_available_memory_base( - tree: GlobalDeviceTree, size_bytes: int, alignment: int = 0x1000, use_iomem: bool = True -) -> Optional[int]: +def chunk_containing( + tree: GlobalDeviceTree, base: int, size: int +) -> Optional[PoolMemoryRegion]: """ - Find available memory region for allocation. + Find the pool chunk that holds a whole memory region. Args: - tree: GlobalDeviceTree to analyze (for pool boundaries) - size_bytes: Size of memory region needed - alignment: Required alignment (default 4KB) - use_iomem: If True, read actual allocations from /proc/iomem (kernel source of truth). - If False, use allocations from tree (for validation/dry-run). + tree: GlobalDeviceTree carrying the live pool chunks + base: Base address of the region + size: Size of the region in bytes Returns: - Base address for allocation, or None if no space available + The chunk containing the region, or None if no single chunk holds it """ - pool_base = tree.hardware.memory.memory_pool_base - pool_end = tree.hardware.memory.memory_pool_end - - if use_iomem: - allocated_regions = get_allocated_memory_regions_from_iomem() - else: - allocated_regions = get_allocated_memory_regions(tree) - # Sort by base address - allocated_regions.sort() - - # Try to find gap between allocations or at start/end - if not allocated_regions: - # No allocations yet, use start of pool (aligned) - aligned_base = (pool_base + alignment - 1) // alignment * alignment - if aligned_base + size_bytes <= pool_end: - return aligned_base - return None - - # Check gap at start - first_base = allocated_regions[0][0] - aligned_base = (pool_base + alignment - 1) // alignment * alignment - if aligned_base + size_bytes <= first_base: - return aligned_base - - # Check gaps between allocations - for i in range(len(allocated_regions) - 1): - current_end = allocated_regions[i][0] + allocated_regions[i][1] - next_base = allocated_regions[i + 1][0] - - # Align current_end - aligned_base = (current_end + alignment - 1) // alignment * alignment - - if aligned_base + size_bytes <= next_base: - return aligned_base - - # Check gap at end - last_end = allocated_regions[-1][0] + allocated_regions[-1][1] - aligned_base = (last_end + alignment - 1) // alignment * alignment - if aligned_base + size_bytes <= pool_end: - return aligned_base - + for chunk in tree.hardware.memory.regions: + if chunk.base <= base and base + size <= chunk.base + chunk.size: + return chunk return None @@ -309,35 +310,33 @@ def validate_memory_allocation( (for update operations) Raises: - ResourceError: If memory region is invalid or conflicts + ResourceError: If the region is misaligned, leaves the pool chunk that + holds it, or conflicts with another instance """ - pool_base = tree.hardware.memory.memory_pool_base - pool_end = tree.hardware.memory.memory_pool_end - memory_end = memory_base + memory_bytes + if memory_base % 0x1000 != 0: + raise ResourceError(f"Memory base {hex(memory_base)} is not 4KB-aligned") - # Check memory is within pool - if memory_base < pool_base: - raise ResourceError(f"Memory base {hex(memory_base)} is below pool base {hex(pool_base)}") + memory_end = memory_base + memory_bytes - if memory_end > pool_end: + # The pool is a list of chunks, so a region has to sit inside one of them; + # spanning two chunks means spanning the host memory between them. + chunks = tree.hardware.memory.regions + if chunks and chunk_containing(tree, memory_base, memory_bytes) is None: + listed = ", ".join(f"{hex(c.base)}-{hex(c.base + c.size - 1)}" for c in chunks) raise ResourceError( - f"Memory region extends beyond pool: " - f"{hex(memory_base)}-{hex(memory_end)} vs pool end {hex(pool_end)}" + f"Memory region {hex(memory_base)}-{hex(memory_end)} does not fit in " + f"any pool chunk ({listed})" ) - # Check alignment (4KB) - if memory_base % 0x1000 != 0: - raise ResourceError(f"Memory base {hex(memory_base)} is not 4KB-aligned") - - # Check for overlaps with other instances for instance in tree.instances.values(): if instance.name == exclude_instance: continue inst_base = instance.resources.memory_base + if not inst_base: + continue inst_end = inst_base + instance.resources.memory_bytes - # Check for overlap if not (memory_end <= inst_base or memory_base >= inst_end): raise ResourceError( f"Memory region {hex(memory_base)}-{hex(memory_end)} " diff --git a/src/kerf/runtime.py b/src/kerf/runtime.py index adcaf7c..a4d7820 100644 --- a/src/kerf/runtime.py +++ b/src/kerf/runtime.py @@ -69,18 +69,13 @@ def create_instance(current: GlobalDeviceTree) -> GlobalDeviceTree: # Validate resources (against baseline) validate_cpu_allocation(modified, cpus) - # Find memory base - memory_base = find_available_memory_base(modified, memory) - if not memory_base: - raise ResourceError("No memory available") - - # Create instance + # Create instance; the kernel places its memory in the pool instance = Instance( name=name, id=find_next_instance_id(modified), resources=InstanceResources( cpus=cpus, - memory_base=memory_base, + memory_base=0, memory_bytes=memory, devices=[] ) @@ -173,6 +168,57 @@ def read_baseline(self) -> GlobalDeviceTree: """ return self.baseline_mgr.read_baseline() + def apply_dtbo(self, dtbo_data: bytes) -> str: + """ + Write a compiled overlay to the kernel and check the transaction. + + Args: + dtbo_data: DTBO blob to hand to /sys/fs/multikernel/overlays/new + + Returns: + Transaction ID (from kernel-created directory) + + Raises: + KernelInterfaceError: If the write fails or the kernel reports a + status other than applied + """ + if not self.overlays_new.exists(): + raise KernelInterfaceError(f"Overlay interface not found: {self.overlays_new}") + + try: + with open(self.overlays_new, "wb") as f: + f.write(dtbo_data) + except OSError as e: + raise KernelInterfaceError( + f"Failed to write overlay to {self.overlays_new}: {e}" + ) from e + + tx_id = self._find_latest_transaction() + if not tx_id: + raise KernelInterfaceError( + "Overlay written but kernel did not create transaction directory" + ) + + tx_dir = self.overlays_dir / f"tx_{tx_id}" + try: + status = (tx_dir / "status").read_text(encoding="utf-8").strip() + except OSError: + # An unreadable status may just mean the kernel is still processing. + return tx_id + + if status not in ("applied", "success", "ok"): + error_msg = f"Overlay transaction {tx_id} failed with status: '{status}'" + try: + # The file is named 'instance' but holds the target path of + # the overlay's first fragment. + target = (tx_dir / "instance").read_text(encoding="utf-8").strip() + error_msg += f" (target: {target})" + except OSError: + pass + raise KernelInterfaceError(error_msg) + + return tx_id + def apply_overlay(self, current: GlobalDeviceTree, modified: GlobalDeviceTree) -> str: """ Apply overlay by writing DTBO to kernel. @@ -215,50 +261,7 @@ def apply_overlay(self, current: GlobalDeviceTree, modified: GlobalDeviceTree) - except Exception as e: raise KernelInterfaceError(f"Failed to generate overlay: {e}") from e - try: - if not self.overlays_new.exists(): - raise KernelInterfaceError(f"Overlay interface not found: {self.overlays_new}") - - with open(self.overlays_new, "wb") as f: - f.write(dtbo_data) - - tx_id = self._find_latest_transaction() - if not tx_id: - raise KernelInterfaceError( - "Overlay written but kernel did not create transaction directory" - ) - - # Verify transaction succeeded by checking status - tx_dir = self.overlays_dir / f"tx_{tx_id}" - status_file = tx_dir / "status" - - if status_file.exists(): - try: - with open(status_file, "r", encoding="utf-8") as f: - status = f.read().strip() - if status not in ("applied", "success", "ok"): - error_msg = f"Overlay transaction {tx_id} failed with status: '{status}'" - instance_file = tx_dir / "instance" - if instance_file.exists(): - try: - with open(instance_file, "r", encoding="utf-8") as f: - instance_name = f.read().strip() - error_msg += f" (instance: {instance_name})" - except OSError: - pass - - raise KernelInterfaceError(error_msg) - except OSError: - # If we can't read status, assume it might still be processing - # But warn that we couldn't verify - pass - - return tx_id - - except OSError as e: - raise KernelInterfaceError( - f"Failed to write overlay to {self.overlays_new}: {e}" - ) from e + return self.apply_dtbo(dtbo_data) def apply_removal_overlay(self, instance_name: str) -> str: """ @@ -278,54 +281,13 @@ def apply_removal_overlay(self, instance_name: str) -> str: Raises: KernelInterfaceError: If overlay application fails """ - with self._acquire_lock(): + with self.lock(): try: dtbo_data = self.overlay_gen.generate_removal_overlay(instance_name) except Exception as e: raise KernelInterfaceError(f"Failed to generate removal overlay: {e}") from e - try: - if not self.overlays_new.exists(): - raise KernelInterfaceError(f"Overlay interface not found: {self.overlays_new}") - - with open(self.overlays_new, "wb") as f: - f.write(dtbo_data) - - tx_id = self._find_latest_transaction() - if not tx_id: - raise KernelInterfaceError( - "Overlay written but kernel did not create transaction directory" - ) - - tx_dir = self.overlays_dir / f"tx_{tx_id}" - status_file = tx_dir / "status" - - if status_file.exists(): - try: - with open(status_file, "r", encoding="utf-8") as f: - status = f.read().strip() - if status not in ("applied", "success", "ok"): - error_msg = ( - f"Overlay transaction {tx_id} failed with status: '{status}'" - ) - instance_file = tx_dir / "instance" - if instance_file.exists(): - try: - with open(instance_file, "r", encoding="utf-8") as f: - tx_instance_name = f.read().strip() - error_msg += f" (instance: {tx_instance_name})" - except OSError: - pass - raise KernelInterfaceError(error_msg) - except OSError: - pass - - return tx_id - - except OSError as e: - raise KernelInterfaceError( - f"Failed to write overlay to {self.overlays_new}: {e}" - ) from e + return self.apply_dtbo(dtbo_data) def _find_latest_transaction(self) -> Optional[str]: """Find the latest transaction ID from kernel-created directories.""" @@ -417,7 +379,7 @@ def list_transactions(self) -> List[Dict[str, str]]: return transactions @contextmanager - def _acquire_lock(self): + def lock(self): """ Acquire file lock for concurrency safety. @@ -486,7 +448,7 @@ def apply_operation(self, operation: Callable[[GlobalDeviceTree], GlobalDeviceTr KernelInterfaceError: If kernel interface operations fail Any exceptions raised by the operation function """ - with self._acquire_lock(): + with self.lock(): current = self.read_baseline() # Apply operation (returns modified state) diff --git a/src/kerf/show/main.py b/src/kerf/show/main.py index 2b02c09..5b11df6 100644 --- a/src/kerf/show/main.py +++ b/src/kerf/show/main.py @@ -33,6 +33,7 @@ from ..exceptions import KernelInterfaceError, ParseError from ..metadata import load_instance_metadata from ..models import GlobalDeviceTree +from ..pool_diff import ANY_NODE from ..resources import get_pool_allocated_bytes from ..utils import get_instance_id_from_name, get_instance_status @@ -278,12 +279,23 @@ def display_baseline_info(tree: GlobalDeviceTree, verbose: bool = False): # CPU Information click.echo("\n CPUs:") click.echo(f" Total: {hardware.cpus.total}") - click.echo( - f" Host Reserved: {len(hardware.cpus.host_reserved)} cpus: {hardware.cpus.host_reserved}" - ) - click.echo( - f" Available: {len(hardware.cpus.available)} cpus: {hardware.cpus.available}" - ) + # A tree read back from the kernel describes the pool, not the host, so + # it carries no host-reserved list worth a line. + if hardware.cpus.host_reserved: + click.echo( + f" Host Reserved: {len(hardware.cpus.host_reserved)} cpus: {hardware.cpus.host_reserved}" + ) + if hardware.cpus.available_free is not None: + click.echo( + f" Pool CPUs: {len(hardware.cpus.available)} cpus: {hardware.cpus.available}" + ) + click.echo( + f" Available CPUs: {len(hardware.cpus.available_free)} cpus: {hardware.cpus.available_free}" + ) + else: + click.echo( + f" Available: {len(hardware.cpus.available)} cpus: {hardware.cpus.available}" + ) if verbose and hardware.cpus.topology: click.echo("\n Topology:") @@ -294,34 +306,39 @@ def display_baseline_info(tree: GlobalDeviceTree, verbose: bool = False): # Memory Information click.echo("\n Memory:") - total_gb = hardware.memory.total_bytes / (1024**3) - reserved_gb = hardware.memory.host_reserved_bytes / (1024**3) - click.echo(f" Total: {total_gb:.2f} GB ({hardware.memory.total_bytes} bytes)") - click.echo( - f" Host Reserved: {reserved_gb:.2f} GB ({hardware.memory.host_reserved_bytes} bytes)" - ) - - # /proc/iomem is the source of truth for the lazy_cma pool and the - # instance allocations carved out of it; the baseline tree only - # snapshots the pool at init time. - usage = get_pool_allocated_bytes() - if usage is not None: - pool_base, pool_bytes, allocated_bytes = usage + # total_bytes/host_reserved_bytes are 0 when the tree only carries pool + # sizes read back from the kernel, not a system-wide total. + if hardware.memory.total_bytes: + total_gb = hardware.memory.total_bytes / (1024**3) + click.echo(f" Total: {total_gb:.2f} GB ({hardware.memory.total_bytes} bytes)") + if hardware.memory.host_reserved_bytes: + reserved_gb = hardware.memory.host_reserved_bytes / (1024**3) + click.echo( + f" Host Reserved: {reserved_gb:.2f} GB ({hardware.memory.host_reserved_bytes} bytes)" + ) + + # /proc/iomem is the source of truth for the pool chunks and the + # instance allocations carved out of them; the baseline tree only + # snapshots the pool as of the last transaction. + click.echo("\n Memory Pool:") + if not hardware.memory.regions: + click.echo(" No memory pool configured") else: - pool_base = hardware.memory.memory_pool_base - pool_bytes = hardware.memory.memory_pool_bytes - allocated_bytes = None - - pool_gb = pool_bytes / (1024**3) - click.echo(f" Pool Base: 0x{pool_base:x}") - click.echo(f" Pool Size: {pool_gb:.2f} GB ({pool_bytes} bytes)") - click.echo(f" Pool End: 0x{pool_base + pool_bytes:x}") - if allocated_bytes is not None: - available_bytes = pool_bytes - allocated_bytes - allocated_gb = allocated_bytes / (1024**3) - available_gb = available_bytes / (1024**3) - click.echo(f" Pool Allocated: {allocated_gb:.2f} GB ({allocated_bytes} bytes)") - click.echo(f" Pool Available: {available_gb:.2f} GB ({available_bytes} bytes)") + for region in hardware.memory.regions: + node = "" if region.node == ANY_NODE else f" node {region.node}" + click.echo( + f" Chunk: {hex(region.base)} " + f"{region.size / (1024**3):.2f} GB{node}" + ) + + usage = get_pool_allocated_bytes() + if usage is not None: + _, pool_bytes, allocated_bytes = usage + available_bytes = pool_bytes - allocated_bytes + allocated_gb = allocated_bytes / (1024**3) + available_gb = available_bytes / (1024**3) + click.echo(f" Pool Allocated: {allocated_gb:.2f} GB ({allocated_bytes} bytes)") + click.echo(f" Pool Available: {available_gb:.2f} GB ({available_bytes} bytes)") # NUMA Topology if hardware.topology and hardware.topology.numa_nodes: @@ -556,10 +573,15 @@ def show(name: Optional[str], verbose: bool): baseline_manager = BaselineManager() try: tree = baseline_manager.read_baseline() - display_baseline_info(tree, verbose) - except (KernelInterfaceError, ParseError) as e: + except ParseError: + # Once the pool is torn down /resources describes the host and + # carries no pool branch to parse. + click.echo("\nNo memory pool configured") + except KernelInterfaceError as e: if verbose: click.echo(f"\nWarning: Could not read baseline: {e}", err=True) + else: + display_baseline_info(tree, verbose) if not instance_names: click.echo("\n" + "=" * 80) diff --git a/src/kerf/topology.py b/src/kerf/topology.py new file mode 100644 index 0000000..1537b16 --- /dev/null +++ b/src/kerf/topology.py @@ -0,0 +1,139 @@ +# Copyright 2026 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Host NUMA topology, as sysfs and /proc/cpuinfo describe it.""" + +import re +from pathlib import Path +from typing import Dict, Iterable, List, Optional + +NODE_ROOT = "/sys/devices/system/node" +CPUINFO = "/proc/cpuinfo" + +_NODE_DIR = re.compile(r"^node(\d+)$") + + +def _parse_cpulist(text: str) -> List[int]: + """Expand a sysfs cpulist ("0-3,8") into logical CPU numbers.""" + cpus: List[int] = [] + for part in text.strip().split(","): + part = part.strip() + if not part: + continue + try: + if "-" in part: + start, _, end = part.partition("-") + cpus.extend(range(int(start), int(end) + 1)) + else: + cpus.append(int(part)) + except ValueError: + continue + return cpus + + +def logical_to_apic(cpuinfo_path: str = CPUINFO) -> Dict[int, int]: + """ + Map logical CPU number to APIC id. + + Args: + cpuinfo_path: Path to read instead of /proc/cpuinfo + + Returns: + Mapping of logical CPU number to APIC id, empty if unreadable + """ + mapping: Dict[int, int] = {} + processor: Optional[int] = None + try: + with open(cpuinfo_path, "r", encoding="utf-8") as f: + for line in f: + key, sep, value = line.partition(":") + if not sep: + continue + key, value = key.strip(), value.strip() + if key == "processor": + processor = _int_or_none(value) + elif key == "apicid" and processor is not None: + apic = _int_or_none(value) + if apic is not None: + mapping[processor] = apic + processor = None + except OSError: + return {} + return mapping + + +def _int_or_none(value: str) -> Optional[int]: + try: + return int(value) + except ValueError: + return None + + +def cpu_numa_nodes(node_root: str = NODE_ROOT, cpuinfo_path: str = CPUINFO) -> Dict[int, int]: + """ + Map APIC id to NUMA node for every CPU the host still reports. + + Args: + node_root: Path to read instead of /sys/devices/system/node + cpuinfo_path: Path to read instead of /proc/cpuinfo + + Returns: + Mapping of APIC id to NUMA node, empty if the topology is unreadable + """ + apic_of = logical_to_apic(cpuinfo_path) + if not apic_of: + return {} + + try: + entries = sorted(Path(node_root).iterdir()) + except OSError: + return {} + + nodes: Dict[int, int] = {} + for entry in entries: + match = _NODE_DIR.match(entry.name) + if not match: + continue + try: + cpulist = (entry / "cpulist").read_text(encoding="utf-8") + except OSError: + continue + for cpu in _parse_cpulist(cpulist): + apic = apic_of.get(cpu) + if apic is not None: + nodes[apic] = int(match.group(1)) + return nodes + + +def node_for_cpus(apic_ids: Iterable[int], mapping: Dict[int, int]) -> Optional[int]: + """ + Pick the NUMA node these CPUs belong to. + + A request has to name one node, so CPUs spread over several follow the + lowest APIC id rather than leaving the choice to the kernel. + + Args: + apic_ids: APIC ids of the requested CPUs + mapping: APIC id to NUMA node mapping from cpu_numa_nodes() + + Returns: + The chosen node, or None if no requested CPU has a known node + """ + known = sorted(apic for apic in apic_ids if apic in mapping) + if not known: + return None + nodes = {mapping[apic] for apic in known} + if len(nodes) == 1: + return nodes.pop() + return mapping[known[0]] diff --git a/src/kerf/update/main.py b/src/kerf/update/main.py index 64ba0b1..682a44c 100644 --- a/src/kerf/update/main.py +++ b/src/kerf/update/main.py @@ -32,7 +32,7 @@ from ..create.main import parse_cpu_spec, parse_memory_base, parse_memory_spec from ..exceptions import KernelInterfaceError, ParseError, ResourceError, ValidationError from ..resources import ( - find_available_memory_base, + chunk_containing, validate_cpu_allocation, validate_memory_allocation, ) @@ -281,35 +281,23 @@ def update_instance_operation(current): if memory_bytes is not None: if memory_base_addr is None: - # Keep the same base address and extend/shrink in place old_base = existing_instance.resources.memory_base old_size = existing_instance.resources.memory_bytes + # The overlay names an existing range, so an instance can + # only grow into the chunk it already sits in. if memory_bytes > old_size: - # Growing: validate the extension region doesn't overlap - extension_base = old_base + old_size - extension_size = memory_bytes - old_size - try: - validate_memory_allocation( - modified, extension_base, extension_size, - exclude_instance=instance_node_name + if chunk_containing(modified, old_base, memory_bytes) is None: + raise ResourceError( + f"Cannot grow instance '{name}' to {memory_bytes} bytes: " + f"the extension leaves the pool chunk holding " + f"{hex(old_base)}-{hex(old_base + old_size - 1)}" ) - memory_base_addr = old_base - except (ResourceError,) as exc: - # Extension conflicts, find a completely new region - found_base = find_available_memory_base(modified, memory_bytes) - if found_base is None: - raise ResourceError( - f"No available memory region found for {memory_bytes} bytes. " - "Try specifying --memory-base or reduce memory size." - ) from exc - memory_base_addr = found_base - elif memory_bytes < old_size: - # Shrinking: always keep the same base - memory_base_addr = old_base - else: - # Same size, no change - memory_base_addr = old_base + validate_memory_allocation( + modified, old_base + old_size, memory_bytes - old_size, + exclude_instance=instance_node_name + ) + memory_base_addr = old_base else: validate_memory_allocation( modified, memory_base_addr, memory_bytes, exclude_instance=instance_node_name @@ -364,50 +352,11 @@ def apply_update_operation(current): dtbo_data = manager.overlay_gen.generate_update_overlay(name, old_instance, new_instance) return dtbo_data - with manager._acquire_lock(): # pylint: disable=protected-access + with manager.lock(): current = manager.read_baseline() dtbo_data = apply_update_operation(current) - try: - if not manager.overlays_new.exists(): - raise KernelInterfaceError( - f"Overlay interface not found: {manager.overlays_new}" - ) - - with open(manager.overlays_new, 'wb') as f: - f.write(dtbo_data) - - tx_id = manager._find_latest_transaction() # pylint: disable=protected-access - if not tx_id: - raise KernelInterfaceError( - "Overlay written but kernel did not create transaction directory" - ) - - tx_dir = manager.overlays_dir / f"tx_{tx_id}" - status_file = tx_dir / "status" - - if status_file.exists(): - try: - with open(status_file, 'r', encoding='utf-8') as f: - status = f.read().strip() - if status not in ("applied", "success", "ok"): - error_msg = f"Overlay transaction {tx_id} failed with status: '{status}'" - instance_file = tx_dir / "instance" - if instance_file.exists(): - try: - with open(instance_file, 'r', encoding='utf-8') as f: - instance_name_from_tx = f.read().strip() - error_msg += f" (instance: {instance_name_from_tx})" - except OSError: - pass - raise KernelInterfaceError(error_msg) - except OSError: - pass - - except OSError as e: - raise KernelInterfaceError( - f"Failed to write overlay to {manager.overlays_new}: {e}" - ) from e + tx_id = manager.apply_dtbo(dtbo_data) click.echo(f"✓ Updated instance '{name}' (transaction {tx_id})") if verbose: diff --git a/tests/conftest.py b/tests/conftest.py index 087eeaf..12b0918 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,6 +31,7 @@ HardwareInventory, CPUAllocation, MemoryAllocation, + PoolMemoryRegion, DeviceInfo, Instance, InstanceResources, @@ -47,8 +48,7 @@ def sample_hardware(): memory = MemoryAllocation( total_bytes=16 * 1024**3, # 16GB host_reserved_bytes=2 * 1024**3, # 2GB - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, # 14GB + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3, node=0)], # 14GB ) devices = { diff --git a/tests/create_test_dtb.py b/tests/create_test_dtb.py index 60490e8..b63e2d4 100644 --- a/tests/create_test_dtb.py +++ b/tests/create_test_dtb.py @@ -29,6 +29,7 @@ HardwareInventory, CPUAllocation, MemoryAllocation, + PoolMemoryRegion, DeviceInfo, Instance, InstanceResources, @@ -47,8 +48,7 @@ def create_test_tree(): memory = MemoryAllocation( total_bytes=16 * 1024**3, # 16GB host_reserved_bytes=2 * 1024**3, # 2GB - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, # 14GB + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3, node=0)], # 14GB ) devices = { diff --git a/tests/demo.py b/tests/demo.py index e3ee8a6..d236072 100644 --- a/tests/demo.py +++ b/tests/demo.py @@ -29,6 +29,7 @@ HardwareInventory, CPUAllocation, MemoryAllocation, + PoolMemoryRegion, DeviceInfo, Instance, InstanceResources, @@ -50,8 +51,7 @@ def create_demo_system(): memory = MemoryAllocation( total_bytes=16 * 1024**3, # 16GB host_reserved_bytes=2 * 1024**3, # 2GB - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, # 14GB + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3, node=0)], # 14GB ) devices = { diff --git a/tests/test_baseline.py b/tests/test_baseline.py index 1a1a027..c730eb8 100644 --- a/tests/test_baseline.py +++ b/tests/test_baseline.py @@ -60,7 +60,7 @@ def test_validate_baseline_missing_hardware(self): def test_validate_baseline_missing_cpus(self): """Test baseline validation fails without CPU info.""" - from kerf.models import GlobalDeviceTree, HardwareInventory, MemoryAllocation + from kerf.models import GlobalDeviceTree, HardwareInventory, MemoryAllocation, PoolMemoryRegion tree = GlobalDeviceTree( hardware=HardwareInventory( @@ -68,8 +68,7 @@ def test_validate_baseline_missing_cpus(self): memory=MemoryAllocation( total_bytes=16 * 1024**3, host_reserved_bytes=2 * 1024**3, - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3, node=0)], ), devices={}, ), @@ -102,9 +101,12 @@ def test_write_and_read_baseline(self, sample_hardware): # Verify assert read_tree.hardware.cpus.available == sample_hardware.cpus.available assert ( - read_tree.hardware.memory.memory_pool_base - == sample_hardware.memory.memory_pool_base + read_tree.hardware.memory.memory_pool_bytes + == sample_hardware.memory.memory_pool_bytes ) + assert read_tree.hardware.memory.requested == { + 0: sample_hardware.memory.memory_pool_bytes + } assert len(read_tree.instances) == 0 finally: # Cleanup @@ -132,3 +134,26 @@ def test_write_baseline_invalid_tree(self, sample_tree): finally: if os.path.exists(baseline_path): os.unlink(baseline_path) + + def test_baseline_emits_memory_requests(self): + import libfdt + from kerf.models import GlobalDeviceTree, HardwareInventory, CPUAllocation, MemoryAllocation + from kerf.dtc.extractor import InstanceExtractor + + hw = HardwareInventory( + cpus=CPUAllocation(total=8, host_reserved=[0, 1], available=[4, 5]), + memory=MemoryAllocation(total_bytes=0, host_reserved_bytes=0, requested={0: 1 << 30, -1: 1 << 29}), + devices={}, + ) + dtb = InstanceExtractor().generate_global_dtb(GlobalDeviceTree(hardware=hw, instances={}, device_references={})) + fdt = libfdt.Fdt(dtb) + res = fdt.path_offset("/resources") + m0 = fdt.subnode_offset(res, "memory@0") + assert fdt.getprop(m0, "size").as_uint64() == 1 << 30 + assert fdt.getprop(m0, "numa-node-id").as_uint32() == 0 + m1 = fdt.subnode_offset(res, "memory@1") + assert fdt.getprop(m1, "size").as_uint64() == 1 << 29 + with pytest.raises(libfdt.FdtException): + fdt.getprop(m1, "numa-node-id") + with pytest.raises(libfdt.FdtException): + fdt.getprop(res, "memory-base") diff --git a/tests/test_init_memory_spec.py b/tests/test_init_memory_spec.py new file mode 100644 index 0000000..d3749ac --- /dev/null +++ b/tests/test_init_memory_spec.py @@ -0,0 +1,101 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The resource specifications accepted by 'kerf init'.""" + +import pytest + +from kerf.init.main import ( + parse_cpu_request, + parse_device_request, + parse_memory_request, + validate_memory_request, +) + +GB = 1 << 30 + + +def test_plain_size_is_any_node(): + assert parse_memory_request("2GB") == {-1: 2 * GB} + + +def test_per_node_sizes(): + assert parse_memory_request("8GB@0, 512MB@1") == {0: 8 * GB, 1: 512 << 20} + + +@pytest.mark.parametrize( + "spec", + ["8GB@x", "8GB@-1", "8GB@0,8GB", "8GB@0,4GB@0", "@0", "8GB@", ""], +) +def test_invalid_specs(spec): + with pytest.raises(ValueError): + parse_memory_request(spec) + + +@pytest.mark.parametrize("spec", ["0@0", "4097", "5000@1"]) +def test_sizes_must_be_positive_and_page_aligned(spec): + with pytest.raises(ValueError): + parse_memory_request(spec) + + +@pytest.mark.parametrize("requested", [{-1: 0}, {0: -4096}, {0: 4097}, {1: 5000}]) +def test_input_file_sizes_get_the_same_check(requested): + # A DTS/DTB request reaches the kernel without going through --memory. + with pytest.raises(ValueError): + validate_memory_request(requested) + + +def test_valid_input_file_sizes_pass(): + validate_memory_request({0: GB, 1: 512 << 20}) + + +@pytest.mark.parametrize("spec", ["none", "NONE", " none ", "0"]) +def test_memory_none_asks_for_nothing(spec): + assert not parse_memory_request(spec) + + +@pytest.mark.parametrize("spec", ["none,1GB@0", "1GB@0,none", "0,2GB", "none,none"]) +def test_memory_none_cannot_be_mixed(spec): + with pytest.raises(ValueError, match="cannot be combined with other entries"): + parse_memory_request(spec) + + +@pytest.mark.parametrize("spec", ["none", "NONE", " none "]) +def test_cpus_none_asks_for_nothing(spec): + assert parse_cpu_request(spec) == [] + + +def test_cpus_are_still_parsed(): + assert parse_cpu_request("1-3,8") == [1, 2, 3, 8] + + +@pytest.mark.parametrize("spec", ["none,4", "4,none", "none,0"]) +def test_cpus_none_cannot_be_mixed(spec): + with pytest.raises(ValueError, match="cannot be combined with other entries"): + parse_cpu_request(spec) + + +@pytest.mark.parametrize("spec", ["none", "NONE", None, ""]) +def test_devices_none_asks_for_nothing(spec): + assert parse_device_request(spec) == [] + + +def test_devices_are_still_parsed(): + assert parse_device_request("enp9s0_dev, nvme0") == ["enp9s0_dev", "nvme0"] + + +@pytest.mark.parametrize("spec", ["none,nvme0", "nvme0,none"]) +def test_devices_none_cannot_be_mixed(spec): + with pytest.raises(ValueError, match="cannot be combined with other entries"): + parse_device_request(spec) diff --git a/tests/test_init_numa_node.py b/tests/test_init_numa_node.py new file mode 100644 index 0000000..e8fb024 --- /dev/null +++ b/tests/test_init_numa_node.py @@ -0,0 +1,212 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The NUMA node 'kerf init' picks for a memory request that names none.""" + +import pytest +from click.testing import CliRunner + +from kerf.dtc.overlay import OverlayGenerator +from kerf.dtc.parser import DeviceTreeParser +from kerf.exceptions import ParseError +from kerf.init import main +from kerf.models import ( + CPUAllocation, + GlobalDeviceTree, + HardwareInventory, + MemoryAllocation, + PoolMemoryRegion, +) +from kerf.pool_diff import ANY_NODE, compute_pool_diff + +MB = 1 << 20 +GB = 1 << 30 + + +def _live_pool(cpus, regions): + """What the kernel reports for a pool that already holds these resources.""" + return GlobalDeviceTree( + hardware=HardwareInventory( + cpus=CPUAllocation(total=16, host_reserved=[0], available=cpus), + memory=MemoryAllocation(total_bytes=0, host_reserved_bytes=0, regions=regions), + devices={}, + ), + instances={}, + device_references={}, + ) + + +@pytest.fixture(name="topology") +def topology_fixture(monkeypatch): + """Let a test state the APIC id to node mapping the host reports.""" + def install(mapping): + monkeypatch.setattr(main, "cpu_numa_nodes", lambda: dict(mapping)) + install({}) + return install + + +def test_cpus_on_one_node_pin_the_request_there(topology): + topology({1: 1, 2: 1, 3: 1}) + + requested, note = main.resolve_memory_nodes({ANY_NODE: 512 * MB}, [1, 2, 3]) + + assert requested == {1: 512 * MB} + assert note == "Memory: 512 MB on node 1 (from CPUs 1-3)" + + +def test_cpus_split_across_nodes_follow_the_lowest_apic_id(topology): + topology({2: 1, 5: 0}) + + requested, note = main.resolve_memory_nodes({ANY_NODE: GB}, [5, 2]) + + assert requested == {1: GB} + assert "from CPUs 2,5" in note + + +def test_pool_cpus_fall_back_to_the_chunks_they_run_on(topology): + # A CPU the pool already holds is offline, so the topology cannot place it. + topology({0: 0}) + regions = [PoolMemoryRegion(0x1_0000_0000, GB, 1)] + + requested, note = main.resolve_memory_nodes( + {ANY_NODE: GB}, [1, 2, 3], pool_cpus={1, 2, 3}, pool_regions=regions) + + assert requested == {1: GB} + assert "chunks the pool already holds" in note + + +def test_chunks_on_several_nodes_do_not_decide(topology): + topology({0: 0}) + regions = [PoolMemoryRegion(0x1_0000_0000, GB, 0), + PoolMemoryRegion(0x2_0000_0000, GB, 1)] + + requested, note = main.resolve_memory_nodes( + {ANY_NODE: GB}, [1, 2], pool_cpus={1, 2}, pool_regions=regions) + + assert requested == {0: GB} + assert "no NUMA topology available" in note + + +def test_no_numa_information_defaults_to_node_zero(topology): + topology({}) + + requested, note = main.resolve_memory_nodes({ANY_NODE: 512 * MB}, [1, 2, 3]) + + assert requested == {0: 512 * MB} + assert note == "Memory: 512 MB on node 0 (no NUMA topology available, defaulting to node 0)" + + +def test_explicit_nodes_are_left_alone(topology): + topology({1: 1}) + + requested, note = main.resolve_memory_nodes({0: GB, 1: 2 * GB}, [1]) + + assert requested == {0: GB, 1: 2 * GB} + assert note is None + + +def test_a_resolved_size_joins_an_explicit_request_on_the_same_node(topology): + topology({4: 0}) + + requested, _ = main.resolve_memory_nodes({0: GB, ANY_NODE: GB}, [4]) + + assert requested == {0: 2 * GB} + + +def test_the_request_built_from_the_command_line_names_a_node(topology, monkeypatch): + topology({0: 0, 1: 1, 2: 1, 3: 1}) + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1, 2, 3}) + + tree = main.build_baseline_from_cmdline("1-3", memory="512MB") + + assert tree.hardware.memory.requested == {1: 512 * MB} + + +def test_the_diff_sees_an_explicit_node(topology, monkeypatch): + topology({0: 0, 1: 1, 2: 1, 3: 1}) + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1, 2, 3}) + current = _live_pool([1, 2, 3], [PoolMemoryRegion(0x1_0000_0000, GB, 1)]) + + requested = main.build_baseline_from_cmdline("1-3", memory="2GB", + pool_cpus=main.pool_apic_ids(current)) + diff = compute_pool_diff(current, requested) + + assert diff.memory_to_pool == [(1, GB)] + + +_DTS_UNPINNED = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + + memory@0 { + size = <0x0 0x40000000>; + }; + }; +}; +""" + + +def test_a_baseline_file_without_numa_node_id_is_resolved_too(topology): + topology({4: 1, 5: 1}) + tree = DeviceTreeParser().parse_dts(_DTS_UNPINNED) + assert tree.hardware.memory.requested == {ANY_NODE: GB} + + requested, note = main.resolve_memory_nodes( + tree.hardware.memory.requested, tree.hardware.cpus.available) + + assert requested == {1: GB} + assert "from CPUs 4-5" in note + + +class _FakeBaselineManager: + """Stands in for BaselineManager: no pool yet, records the baseline write.""" + + def __init__(self): + self.written = [] + + def validate_baseline(self, tree): + """Accept anything: the baseline shape is covered elsewhere.""" + + def read_baseline(self): + raise ParseError("No memory description in /resources") + + def write_baseline(self, tree): + self.written.append(tree) + + +class _FakeManager: + """Stands in for DeviceTreeManager: never touches sysfs.""" + + def __init__(self, *_args, **_kwargs): + self.overlay_gen = OverlayGenerator() + + +def test_init_resolves_a_baseline_file_before_writing_it(topology, monkeypatch, tmp_path): + topology({4: 1, 5: 1}) + baseline_mgr = _FakeBaselineManager() + monkeypatch.setattr(main, "mount_multikernel_fs", lambda verbose=False: None) + monkeypatch.setattr(main, "DeviceTreeManager", _FakeManager) + monkeypatch.setattr(main, "get_busy_chunks_from_iomem", set) + monkeypatch.setattr(main, "BaselineManager", lambda *a, **kw: baseline_mgr) + dts = tmp_path / "baseline.dts" + dts.write_text(_DTS_UNPINNED, encoding="utf-8") + + result = CliRunner().invoke(main.init, [f"--input={dts}"]) + + assert result.exit_code == 0, result.output + assert "Memory: 1024 MB on node 1 (from CPUs 4-5)" in result.output + assert baseline_mgr.written[-1].hardware.memory.requested == {1: GB} diff --git a/tests/test_init_reconcile.py b/tests/test_init_reconcile.py new file mode 100644 index 0000000..02d7845 --- /dev/null +++ b/tests/test_init_reconcile.py @@ -0,0 +1,323 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The decision 'kerf init' makes between writing a baseline and applying a diff.""" + +import contextlib + +import libfdt +import pytest + +from kerf.dtc.overlay import OverlayGenerator +from kerf.exceptions import ParseError, ValidationError +from kerf.init import main +from kerf.init.main import reconcile_pool +from kerf.models import ( + CPUAllocation, + GlobalDeviceTree, + HardwareInventory, + MemoryAllocation, + PoolMemoryRegion, +) + +GB = 1 << 30 + + +class FakeManager: + """Stands in for DeviceTreeManager: records the blob, never touches sysfs.""" + + def __init__(self): + self.overlay_gen = OverlayGenerator() + self.applied = [] + + @contextlib.contextmanager + def lock(self): + yield + + def apply_dtbo(self, dtbo_data): + self.applied.append(bytes(dtbo_data)) + return "1" + + +class FakeBaselineManager: + """Stands in for BaselineManager: records writes, replays one read-back.""" + + def __init__(self, live=None, read_error=None): + self.written = [] + self.live = live + self.read_error = read_error + + def write_baseline(self, tree): + self.written.append(tree) + + def read_baseline(self): + if self.read_error is not None: + raise self.read_error + return self.live + + +def _tree(cpus, regions, requested, available_free=None): + return GlobalDeviceTree( + hardware=HardwareInventory( + cpus=CPUAllocation(total=16, host_reserved=[0], available=cpus, + available_free=available_free), + memory=MemoryAllocation( + total_bytes=8 * GB, + host_reserved_bytes=0, + regions=regions, + requested=requested, + ), + devices={}, + ), + instances={}, + device_references={}, + ) + + +def test_first_init_writes_the_baseline(): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + requested = _tree([4, 5], [], {-1: 2 * GB}) + + assert reconcile_pool(None, requested, set(), False, manager, baseline_mgr) is None + assert baseline_mgr.written == [requested] + assert not manager.applied + + +def test_first_init_dry_run_writes_nothing(): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + empty = _tree([], [], {}) + + assert reconcile_pool(empty, _tree([4], [], {-1: GB}), set(), True, manager, baseline_mgr) is None + assert not baseline_mgr.written + assert not manager.applied + + +def test_matching_pool_is_a_no_op(): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + current = _tree([4, 5], [PoolMemoryRegion(0x1_0000_0000, 2 * GB, 0)], {}) + requested = _tree([4, 5], [], {0: 2 * GB}) + + diff = reconcile_pool(current, requested, set(), False, manager, baseline_mgr) + + assert diff.is_empty() + assert not manager.applied + assert not baseline_mgr.written + + +def test_dry_run_reports_without_applying(): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + current = _tree([4, 5], [], {}, available_free=[4, 5]) + requested = _tree([4, 5, 6], [], {0: GB}) + + diff = reconcile_pool(current, requested, set(), True, manager, baseline_mgr) + + assert diff.cpus_to_pool == [6] + assert diff.memory_to_pool == [(0, GB)] + assert not manager.applied + + +def test_apply_writes_a_pool_overlay(): + manager = FakeManager() + current = _tree([4, 5], [PoolMemoryRegion(0x1_0000_0000, GB, 1)], {}) + requested = _tree([4], [], {0: GB}) + baseline_mgr = FakeBaselineManager(live=_tree([4], [PoolMemoryRegion(0x2_0000_0000, GB, 0)], {})) + + diff = reconcile_pool(current, requested, set(), False, manager, baseline_mgr) + + assert diff.cpus_to_host == [5] + assert len(manager.applied) == 1 + fdt = libfdt.Fdt(manager.applied[0]) + overlay = fdt.path_offset("/fragment@0/__overlay__") + assert fdt.getprop(fdt.path_offset("/fragment@0"), "target-path").as_str() == "/resources" + assert fdt.subnode_offset(overlay, "cpu-remove") >= 0 + assert fdt.subnode_offset(overlay, "memory-add") >= 0 + assert fdt.subnode_offset(overlay, "memory-remove") >= 0 + assert not baseline_mgr.written + + +def test_an_empty_request_returns_everything(monkeypatch): + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1, 4, 5}) + monkeypatch.setattr(main, "list_instance_names", lambda: []) + manager, baseline_mgr = FakeManager(), FakeBaselineManager(live=_tree([], [], {})) + current = _tree([4, 5], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + requested = main.build_baseline_from_cmdline("none", memory="none") + + diff = main.reconcile_pool(current, requested, set(), False, manager, baseline_mgr) + + assert diff.cpus_to_host == [4, 5] + assert [r.base for r in diff.memory_to_host] == [0x1_0000_0000] + assert diff.memory_to_pool == [] + assert len(manager.applied) == 1 + + +def test_an_empty_request_dry_run_shows_the_releases(monkeypatch, capsys): + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 4, 5}) + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + current = _tree([4, 5], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + requested = main.build_baseline_from_cmdline("none", memory="none") + + main.reconcile_pool(current, requested, set(), True, manager, baseline_mgr) + out = capsys.readouterr().out + + assert "CPUs to host: 4, 5" in out + assert "Memory to host: 0x100000000 (1024 MB)" in out + assert not manager.applied + assert not baseline_mgr.written + + +def test_unparseable_read_back_is_a_first_init(): + manager = FakeManager() + baseline_mgr = FakeBaselineManager( + read_error=ParseError("No memory description in /resources")) + requested = _tree([4, 5], [], {-1: GB}) + + current = main.read_current_pool(baseline_mgr) + + assert current is None + assert reconcile_pool(current, requested, set(), False, manager, baseline_mgr) is None + assert baseline_mgr.written == [requested] + assert not manager.applied + + +def test_host_cpu_list_alone_is_not_a_live_pool(): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + # What the kernel publishes before a pool exists: its own online CPUs, + # no cpus-available, no memory@N. + current = _tree([0, 1, 2, 3], [], {}) + requested = _tree([4, 5], [], {-1: GB}) + + assert not main.pool_is_live(current) + assert reconcile_pool(current, requested, set(), False, manager, baseline_mgr) is None + assert baseline_mgr.written == [requested] + assert not manager.applied + + +def test_an_empty_request_against_an_empty_pool_writes_nothing(monkeypatch): + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1}) + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + requested = main.build_baseline_from_cmdline("none", memory="none") + + assert reconcile_pool(_tree([0, 1], [], {}), requested, set(), False, + manager, baseline_mgr) is None + assert not baseline_mgr.written + assert not manager.applied + + +def test_an_empty_request_refuses_while_instances_exist(monkeypatch): + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 4}) + monkeypatch.setattr(main, "list_instance_names", lambda: ["db", "web"]) + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + current = _tree([4], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + requested = main.build_baseline_from_cmdline("none", memory="none") + + with pytest.raises(ValidationError, match="delete instances db, web first"): + reconcile_pool(current, requested, set(), False, manager, baseline_mgr) + assert not manager.applied + + +def test_unshrinkable_surplus_is_reported(capsys): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + current = _tree([4, 5], [PoolMemoryRegion(0x1_0000_0000, 2 * GB, 0)], {}) + requested = _tree([4, 5], [], {0: GB}) + + diff = reconcile_pool(current, requested, set(), False, manager, baseline_mgr) + + assert diff.is_empty() + assert not manager.applied + err = capsys.readouterr().err + assert "node 0 still holds 1024 MB more than requested" in err + assert "only whole idle chunks can be returned" in err + + +def test_dry_run_reports_the_surplus_too(capsys): + manager, baseline_mgr = FakeManager(), FakeBaselineManager() + current = _tree([4], [PoolMemoryRegion(0x1_0000_0000, 2 * GB, 0)], {}) + requested = _tree([4, 5], [], {0: GB}) + + reconcile_pool(current, requested, set(), True, manager, baseline_mgr) + + assert "still holds 1024 MB more than requested" in capsys.readouterr().err + + +def test_cpu_only_pool_grows_memory(): + # A pool that kept its CPUs but gave back every chunk is live, and the + # request only has to add the memory back. + manager = FakeManager() + current = _tree([1, 2, 3], [], {}, available_free=[1, 2, 3]) + requested = _tree([1, 2, 3], [], {-1: GB // 2}) + baseline_mgr = FakeBaselineManager( + live=_tree([1, 2, 3], [PoolMemoryRegion(0x1_0000_0000, GB // 2, 0)], {})) + + diff = reconcile_pool(current, requested, set(), False, manager, baseline_mgr) + + assert diff.memory_to_pool == [(-1, GB // 2)] + assert diff.cpus_to_pool == [] and diff.cpus_to_host == [] + assert diff.memory_to_host == [] + assert len(manager.applied) == 1 + assert not baseline_mgr.written + + +def test_pool_cpus_stay_valid_apic_ids(monkeypatch): + # A CPU in the pool is gone from /proc/cpuinfo, but re-init must still name it. + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0}) + live = _tree([1, 2, 3], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + + tree = main.build_baseline_from_cmdline( + "1-3", memory="512MB", pool_cpus=main.pool_apic_ids(live)) + + assert tree.hardware.cpus.available == [1, 2, 3] + assert tree.hardware.cpus.host_reserved == [0] + assert tree.hardware.cpus.total == 4 + + +def test_apic_id_outside_system_and_pool_is_rejected(monkeypatch): + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0}) + live = _tree([1], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + + with pytest.raises(ValueError, match=r"Invalid APIC ID\(s\) specified: \[9\]"): + main.build_baseline_from_cmdline("1,9", memory="512MB", + pool_cpus=main.pool_apic_ids(live)) + + +def test_pool_apic_ids_ignores_a_host_only_read_back(): + assert main.pool_apic_ids(None) == set() + assert main.pool_apic_ids(_tree([0, 1, 2, 3], [], {})) == set() + + +def test_an_empty_request_reserves_pool_cpus(monkeypatch): + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0}) + + tree = main.build_baseline_from_cmdline("none", memory="none", pool_cpus={1, 2, 3}) + + assert tree.hardware.cpus.host_reserved == [0, 1, 2, 3] + assert not tree.hardware.cpus.available + assert tree.hardware.cpus.total == 4 + assert not tree.hardware.memory.requested + assert tree.hardware.memory.total_bytes == 0 + + +def test_an_empty_request_does_not_read_the_pool_back(monkeypatch, capsys): + # Once the pool is gone /resources has no memory@N, so a read-back + # always fails; complaining about it makes a clean exit look broken. + monkeypatch.setattr(main, "get_valid_apic_ids_from_system", lambda: {0, 1}) + monkeypatch.setattr(main, "list_instance_names", lambda: []) + manager = FakeManager() + baseline_mgr = FakeBaselineManager( + read_error=ParseError("No memory description in /resources")) + current = _tree([1], [PoolMemoryRegion(0x1_0000_0000, GB, 0)], {}) + requested = main.build_baseline_from_cmdline("none", memory="none", pool_cpus={1}) + + main.reconcile_pool(current, requested, set(), False, manager, baseline_mgr) + + assert "could not read the pool back" not in capsys.readouterr().err diff --git a/tests/test_kerf.py b/tests/test_kerf.py index e5d5621..c38ed31 100644 --- a/tests/test_kerf.py +++ b/tests/test_kerf.py @@ -29,6 +29,7 @@ HardwareInventory, CPUAllocation, MemoryAllocation, + PoolMemoryRegion, DeviceInfo, Instance, InstanceResources, @@ -49,8 +50,7 @@ def create_test_tree(): memory = MemoryAllocation( total_bytes=16 * 1024**3, # 16GB host_reserved_bytes=2 * 1024**3, # 2GB - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, # 14GB + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3, node=0)], # 14GB ) devices = { @@ -133,6 +133,21 @@ def test_validation(): assert result.is_valid, "Validation should pass for valid tree" +def test_reporter_zero_total_bytes(): + """A kernel read-back tree has total_bytes == 0; percentages must not divide by zero.""" + tree = create_test_tree() + tree.hardware.memory.total_bytes = 0 + tree.hardware.memory.host_reserved_bytes = 0 + + validator = MultikernelValidator() + result = validator.validate(tree) + + reporter = ValidationReporter() + report = reporter.generate_report(result, tree, verbose=True) + + assert "n/a" in report + + def test_extraction(): """Test instance extraction functionality.""" print("\n=== Testing Instance Extraction ===") diff --git a/tests/test_models.py b/tests/test_models.py index 39631f9..293da50 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -22,6 +22,7 @@ Instance, InstanceResources, MemoryAllocation, + PoolMemoryRegion, ) @@ -53,8 +54,7 @@ def test_memory_allocation_creation(self): memory = MemoryAllocation( total_bytes=16 * 1024**3, host_reserved_bytes=2 * 1024**3, - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3)], ) assert memory.total_bytes == 16 * 1024**3 @@ -67,14 +67,36 @@ def test_memory_pool_end(self): memory = MemoryAllocation( total_bytes=16 * 1024**3, host_reserved_bytes=2 * 1024**3, - memory_pool_base=0x80000000, - memory_pool_bytes=14 * 1024**3, + regions=[PoolMemoryRegion(base=0x80000000, size=14 * 1024**3)], ) expected_end = 0x80000000 + 14 * 1024**3 assert memory.memory_pool_end == expected_end +def test_memory_allocation_derives_pool_from_regions(): + """Pool base/bytes come from the live chunk list when present.""" + mem = MemoryAllocation( + total_bytes=0, + host_reserved_bytes=0, + regions=[ + PoolMemoryRegion(base=0x100000000, size=1 << 30, node=0), + PoolMemoryRegion(base=0x300000000, size=1 << 29, node=1), + ], + ) + assert mem.memory_pool_base == 0x100000000 + assert mem.memory_pool_bytes == (1 << 30) + (1 << 29) + assert mem.bytes_on_node(1) == 1 << 29 + assert mem.bytes_on_node(2) == 0 + + +def test_memory_allocation_derives_pool_from_requested_when_no_regions(): + """Falls back to the requested per-node sizes when no chunks were read back yet.""" + mem = MemoryAllocation(total_bytes=0, host_reserved_bytes=0, requested={-1: 1 << 30}) + assert mem.memory_pool_base == 0 + assert mem.memory_pool_bytes == 1 << 30 + + class TestDeviceInfo: """Test device information model.""" diff --git a/tests/test_parser.py b/tests/test_parser.py index 3cbdeb3..24ffba4 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -37,10 +37,9 @@ def test_parse_dtb_roundtrip(self, sample_tree): # Verify resources match assert parsed_tree.hardware.cpus.available == sample_tree.hardware.cpus.available - assert ( - parsed_tree.hardware.memory.memory_pool_base - == sample_tree.hardware.memory.memory_pool_base - ) + assert parsed_tree.hardware.memory.requested == { + 0: sample_tree.hardware.memory.memory_pool_bytes + } assert ( parsed_tree.hardware.memory.memory_pool_bytes == sample_tree.hardware.memory.memory_pool_bytes diff --git a/tests/test_parser_pool.py b/tests/test_parser_pool.py new file mode 100644 index 0000000..1917a3f --- /dev/null +++ b/tests/test_parser_pool.py @@ -0,0 +1,288 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for parsing pool chunks and per-node memory requests.""" + +import struct +from pathlib import Path + +import libfdt +import pytest + +from kerf.dtc.parser import DeviceTreeParser +from kerf.exceptions import ParseError + + +def _dtb(build): + sw = libfdt.FdtSw() + sw.finish_reservemap() + sw.begin_node("") + sw.property_string("compatible", "linux,multikernel-host") + sw.begin_node("resources") + sw.property("cpus", struct.pack(">QQ", 4, 5)) + build(sw) + sw.end_node() + sw.end_node() + fdt = sw.as_fdt() + fdt.pack() + return bytes(fdt.as_bytearray()) + + +def test_parse_readback_regions(): + def build(sw): + sw.begin_node("memory@100000000") + sw.property_string("device_type", "memory") + sw.property("reg", struct.pack(">QQ", 0x100000000, 1 << 30)) + sw.property_u32("numa-node-id", 1) + sw.end_node() + + tree = DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + mem = tree.hardware.memory + assert [(r.base, r.size, r.node) for r in mem.regions] == [(0x100000000, 1 << 30, 1)] + assert mem.requested == {} + assert mem.total_bytes == 1 << 30 + + +def test_parse_requested_memory_nodes(): + def build(sw): + sw.begin_node("memory@0") + sw.property_u64("size", 1 << 30) + sw.property_u32("numa-node-id", 0) + sw.end_node() + sw.begin_node("memory@1") + sw.property_u64("size", 1 << 29) + sw.end_node() + + tree = DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + assert tree.hardware.memory.regions == [] + assert tree.hardware.memory.requested == {0: 1 << 30, -1: 1 << 29} + assert tree.hardware.memory.total_bytes == (1 << 30) + (1 << 29) + + +def test_parse_memory_node_without_reg_or_size_rejected(): + def build(sw): + sw.begin_node("memory@0") + sw.property_u32("numa-node-id", 0) + sw.end_node() + + with pytest.raises(ParseError): + DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + + +def test_parse_legacy_memory_base_bytes_rejected(): + def build(sw): + sw.property_u64("memory-base", 0x80000000) + sw.property_u64("memory-bytes", 1 << 30) + + with pytest.raises(ParseError): + DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + + +def test_parse_cpus_available(): + def build(sw): + sw.property("cpus-available", struct.pack(">Q", 5)) + sw.begin_node("memory@0") + sw.property_u64("size", 1 << 30) + sw.end_node() + + tree = DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + assert tree.hardware.cpus.available == [4, 5] + assert tree.hardware.cpus.available_free == [5] + + +def test_parse_cpus_available_absent(): + def build(sw): + sw.begin_node("memory@0") + sw.property_u64("size", 1 << 30) + sw.end_node() + + tree = DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + assert tree.hardware.cpus.available_free is None + + +_DTS_REQUEST = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + + memory@0 { + size = <0x0 0x40000000>; + numa-node-id = <1>; + }; + + memory@1 { + size = <0x0 0x20000000>; + }; + }; +}; +""" + +_DTS_READBACK = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + + memory@100000000 { + device_type = "memory"; + reg = <0x1 0x0 0x0 0x40000000>; + numa-node-id = <1>; + }; + }; +}; +""" + +_DTS_LEGACY = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + memory-base = <0x0 0x80000000>; + memory-bytes = <0x0 0x40000000>; + }; +}; +""" + + +def test_parse_dts_requested_memory_nodes(): + tree = DeviceTreeParser().parse_dts(_DTS_REQUEST) + assert tree.hardware.memory.regions == [] + assert tree.hardware.memory.requested == {1: 1 << 30, -1: 1 << 29} + + +def test_parse_dts_readback_regions(): + tree = DeviceTreeParser().parse_dts(_DTS_READBACK) + regions = tree.hardware.memory.regions + assert [(r.base, r.size, r.node) for r in regions] == [(0x100000000, 1 << 30, 1)] + + +def test_parse_dts_legacy_memory_base_bytes_rejected(): + with pytest.raises(ParseError): + DeviceTreeParser().parse_dts(_DTS_LEGACY) + + +_DTS_TOPOLOGY = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + + topology { + numa-nodes { + node@0 { + node-id = <0>; + memory-base = <0x0 0x0>; + memory-size = <0x0 0x800000000>; + cpus = <4>; + }; + + node@1 { + node-id = <1>; + memory-base = <0x0 0x800000000>; + memory-size = <0x0 0x800000000>; + cpus = <5>; + }; + }; + }; + + memory@0 { + size = <0x0 0x40000000>; + numa-node-id = <1>; + }; + }; +}; +""" + +_DTS_LIVE_NO_MEMORY = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + cpus-available = <5>; + }; +}; +""" + +_DTS_NO_MEMORY = """ +/multikernel-v1/; + +/ { + resources { + cpus = <4 5>; + }; +}; +""" + + +def test_parse_dts_topology_memory_base_is_not_the_pool(): + tree = DeviceTreeParser().parse_dts(_DTS_TOPOLOGY) + assert tree.hardware.memory.requested == {1: 1 << 30} + assert tree.hardware.memory.regions == [] + + +def test_parse_dts_without_memory_rejected(): + with pytest.raises(ParseError, match="No memory description"): + DeviceTreeParser().parse_dts(_DTS_NO_MEMORY) + + +def test_parse_dtb_without_memory_rejected(): + def build(sw): + sw.property_u32("placeholder", 0) + + with pytest.raises(ParseError, match="No memory description"): + DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + + +def test_parse_dts_live_pool_without_memory_accepted(): + tree = DeviceTreeParser().parse_dts(_DTS_LIVE_NO_MEMORY) + assert tree.hardware.memory.regions == [] + assert tree.hardware.memory.requested == {} + assert tree.hardware.memory.total_bytes == 0 + assert tree.hardware.cpus.available_free == [5] + + +def test_parse_dtb_live_pool_without_memory_accepted(): + def build(sw): + sw.property("cpus-available", struct.pack(">Q", 5)) + + tree = DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + assert tree.hardware.memory.regions == [] + assert tree.hardware.memory.requested == {} + assert tree.hardware.memory.total_bytes == 0 + assert tree.hardware.cpus.available_free == [5] + + +def test_parse_dtb_legacy_memory_base_alone_rejected(): + def build(sw): + sw.property_u64("memory-base", 0x80000000) + sw.begin_node("memory@0") + sw.property_u64("size", 1 << 30) + sw.end_node() + + with pytest.raises(ParseError, match="not supported"): + DeviceTreeParser().parse_dtb_from_bytes(_dtb(build)) + + +@pytest.mark.parametrize( + "path", sorted(str(p) for p in Path(__file__).parent.parent.glob("examples/*.dts")) +) +def test_parse_example_dts(path): + tree = DeviceTreeParser().parse_dts(Path(path).read_text(encoding="utf-8")) + assert tree.hardware.memory.memory_pool_bytes > 0 diff --git a/tests/test_pool_diff.py b/tests/test_pool_diff.py new file mode 100644 index 0000000..0360e5f --- /dev/null +++ b/tests/test_pool_diff.py @@ -0,0 +1,115 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Tests for computing the difference between the live pool and a requested baseline. +""" + +from kerf.models import (CPUAllocation, GlobalDeviceTree, HardwareInventory, + MemoryAllocation, PoolMemoryRegion, DeviceInfo) +from kerf.pool_diff import compute_pool_diff + +GB = 1 << 30 + + +def _tree(cpus, regions=None, requested=None, devices=()): + hw = HardwareInventory( + cpus=CPUAllocation(total=16, host_reserved=[0], available=list(cpus)), + memory=MemoryAllocation(total_bytes=0, host_reserved_bytes=0, + regions=list(regions or []), requested=dict(requested or {})), + devices={d: DeviceInfo(name=d, compatible="pci", pci_id=d) for d in devices}, + ) + return GlobalDeviceTree(hardware=hw, instances={}, device_references={}) + + +def test_cpu_and_device_set_differences(): + cur = _tree([4, 5, 6], devices=["0000:03:00.0"]) + req = _tree([5, 6, 7], devices=["0000:04:00.0"]) + d = compute_pool_diff(cur, req) + assert d.cpus_to_pool == [7] + assert d.cpus_to_host == [4] + assert d.devices_to_pool == ["0000:04:00.0"] + assert d.devices_to_host == ["0000:03:00.0"] + + +def test_memory_grow_per_node(): + cur = _tree([4], regions=[PoolMemoryRegion(0x1_0000_0000, GB, 0)]) + req = _tree([4], requested={0: 2 * GB, 1: GB}) + d = compute_pool_diff(cur, req) + assert sorted(d.memory_to_pool) == [(0, GB), (1, GB)] + assert d.memory_to_host == [] + + +def test_memory_shrink_prefers_idle_chunks(): + a = PoolMemoryRegion(0x1_0000_0000, GB, 0) + b = PoolMemoryRegion(0x2_0000_0000, GB, 0) + cur = _tree([4], regions=[a, b]) + req = _tree([4], requested={0: GB}) + d = compute_pool_diff(cur, req, busy_chunks={a.base}) + assert d.memory_to_host == [b] + assert d.memory_to_pool == [] + + +def test_any_node_request_matches_total(): + cur = _tree([4], regions=[PoolMemoryRegion(0x1_0000_0000, GB, 0), PoolMemoryRegion(0x2_0000_0000, GB, 1)]) + req = _tree([4], requested={-1: 2 * GB}) + assert compute_pool_diff(cur, req).is_empty() + + +def test_identical_state_is_empty(): + cur = _tree([4, 5], regions=[PoolMemoryRegion(0x1_0000_0000, GB, 0)]) + req = _tree([4, 5], requested={0: GB}) + assert compute_pool_diff(cur, req).is_empty() + + +def test_unrequested_node_memory_is_surplus(): + a = PoolMemoryRegion(0x1_0000_0000, GB, 0) + cur = _tree([4], regions=[a]) + req = _tree([4], requested={1: GB}) + d = compute_pool_diff(cur, req) + assert d.memory_to_pool == [(1, GB)] + assert d.memory_to_host == [a] + + +def test_shrink_prefers_larger_idle_chunks(): + small = PoolMemoryRegion(0x1_0000_0000, GB // 2, 0) + big = PoolMemoryRegion(0x2_0000_0000, GB, 0) + cur = _tree([4], regions=[small, big]) + req = _tree([4], requested={0: GB // 2}) + d = compute_pool_diff(cur, req) + assert d.memory_to_host == [big] + assert d.memory_to_pool == [] + + +def test_busy_chunks_are_never_offered_to_the_host(): + # The only chunk that would fit the surplus holds a running instance. + idle = PoolMemoryRegion(0x1_0000_0000, GB, 0) + busy = PoolMemoryRegion(0x2_0000_0000, GB // 2, 0) + cur = _tree([4], regions=[idle, busy]) + req = _tree([4], requested={0: GB}) + d = compute_pool_diff(cur, req, busy_chunks={busy.base}) + assert d.memory_to_host == [] + assert d.is_empty() + + +def test_surplus_smaller_than_every_chunk_stays_in_the_pool(): + # Only whole chunks go back to the host, so a 1GB pool asked to shrink to + # 512MB keeps its chunk and the caller has to be told. + chunk = PoolMemoryRegion(0x1_0000_0000, GB, 0) + cur = _tree([4], regions=[chunk]) + req = _tree([4], requested={0: GB // 2}) + d = compute_pool_diff(cur, req) + assert d.memory_to_host == [] + assert d.memory_to_pool == [] + assert d.is_empty() diff --git a/tests/test_pool_overlay.py b/tests/test_pool_overlay.py new file mode 100644 index 0000000..d0ceb2b --- /dev/null +++ b/tests/test_pool_overlay.py @@ -0,0 +1,158 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pool transaction overlays and target-path addressing of instance overlays.""" + +import copy +import struct + +import libfdt + +from kerf.dtc.overlay import OverlayGenerator +from kerf.models import GlobalDeviceTree, PoolMemoryRegion +from kerf.pool_diff import PoolDiff + +GB = 1 << 30 +MISSING = -libfdt.FDT_ERR_NOTFOUND + + +def _ov(dtbo): + fdt = libfdt.Fdt(dtbo) + return fdt, fdt.path_offset("/fragment@0/__overlay__") + + +def _walk(fdt, offset=0): + yield offset + child = fdt.first_subnode(offset, quiet=[libfdt.FDT_ERR_NOTFOUND]) + while child >= 0: + yield from _walk(fdt, child) + child = fdt.next_subnode(child, quiet=[libfdt.FDT_ERR_NOTFOUND]) + + +def _grown(instance, cpus, memory_bytes, numa_nodes=None): + new = copy.deepcopy(instance) + new.resources.cpus = cpus + new.resources.memory_bytes = memory_bytes + new.resources.numa_nodes = numa_nodes + return new + + +def test_pool_overlay_layout(): + diff = PoolDiff( + cpus_to_pool=[8, 9], cpus_to_host=[4], + devices_to_pool=["0000:04:00.0"], devices_to_host=["0000:03:00.0"], + memory_to_pool=[(1, GB), (-1, GB // 2)], + memory_to_host=[PoolMemoryRegion(0x2_0000_0000, GB, 0)], + ) + fdt, ov = _ov(OverlayGenerator().generate_pool_overlay(diff)) + + frag = fdt.path_offset("/fragment@0") + assert fdt.getprop(frag, "target-path").as_str() == "/resources" + grow = fdt.subnode_offset(ov, "memory-add") + m0 = fdt.subnode_offset(grow, "memory@0") + assert fdt.getprop(m0, "size").as_uint64() == GB + assert fdt.getprop(m0, "numa-node-id").as_uint32() == 1 + m1 = fdt.subnode_offset(grow, "memory@1") + assert fdt.getprop(m1, "size").as_uint64() == GB // 2 + assert fdt.getprop(m1, "numa-node-id", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + # The kernel picks the chunk, so a grow item never names an address. + assert fdt.getprop(m0, "reg", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + assert fdt.getprop(m1, "reg", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + + shrink = fdt.subnode_offset(ov, "memory-remove") + reg = bytes(fdt.getprop(fdt.subnode_offset(shrink, "memory@0"), "reg")) + assert struct.unpack(">QQ", reg) == (0x2_0000_0000, GB) + + assert fdt.getprop(fdt.subnode_offset(fdt.subnode_offset(ov, "cpu-add"), "cpu@8"), + "reg").as_uint64() == 8 + assert fdt.getprop(fdt.subnode_offset(fdt.subnode_offset(ov, "cpu-remove"), "cpu@4"), + "reg").as_uint64() == 4 + assert fdt.getprop(fdt.subnode_offset(fdt.subnode_offset(ov, "device-add"), "pci@0"), + "pci-id").as_str() == "0000:04:00.0" + assert fdt.getprop(fdt.subnode_offset(fdt.subnode_offset(ov, "device-remove"), "pci@0"), + "pci-id").as_str() == "0000:03:00.0" + + +def test_pool_overlay_omits_empty_ops(): + fdt, ov = _ov(OverlayGenerator().generate_pool_overlay(PoolDiff(cpus_to_pool=[5]))) + assert fdt.subnode_offset(ov, "memory-remove", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + assert fdt.subnode_offset(ov, "cpu-add") >= 0 + + +def test_update_overlay_targets_the_instance_path(sample_instances): + old = sample_instances["database"] + new = _grown(old, old.resources.cpus + [16], old.resources.memory_bytes + GB) + fdt, ov = _ov(OverlayGenerator().generate_update_overlay("database", old, new)) + + assert fdt.getprop(fdt.path_offset("/fragment@0"), "target-path").as_str() == "/instances/database" + for offset in _walk(fdt): + assert fdt.getprop(offset, "mk,instance", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + + add = fdt.subnode_offset(ov, "memory-add") + reg = bytes(fdt.getprop(fdt.subnode_offset(add, "memory@0"), "reg")) + assert struct.unpack(">QQ", reg) == (old.resources.memory_base + old.resources.memory_bytes, GB) + assert fdt.subnode_offset(add, "region@0", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + + +def test_update_overlay_pins_new_resources_to_the_instance_node(sample_instances): + old = sample_instances["database"] + new = _grown(old, old.resources.cpus + [16], old.resources.memory_bytes + GB, numa_nodes=[1]) + fdt, ov = _ov(OverlayGenerator().generate_update_overlay("database", old, new)) + + cpu = fdt.subnode_offset(fdt.subnode_offset(ov, "cpu-add"), "cpu@16") + assert fdt.getprop(cpu, "numa-node-id").as_uint32() == 1 + assert fdt.getprop(cpu, "numa-node", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + + mem = fdt.subnode_offset(fdt.subnode_offset(ov, "memory-add"), "memory@0") + assert fdt.getprop(mem, "numa-node-id").as_uint32() == 1 + + +def test_update_overlay_shrink_names_memory_items(sample_instances): + old = sample_instances["database"] + new = _grown(old, old.resources.cpus[:-1], old.resources.memory_bytes - GB) + fdt, ov = _ov(OverlayGenerator().generate_update_overlay("database", old, new)) + + remove = fdt.subnode_offset(ov, "memory-remove") + reg = bytes(fdt.getprop(fdt.subnode_offset(remove, "memory@0"), "reg")) + assert struct.unpack(">QQ", reg) == ( + old.resources.memory_base + new.resources.memory_bytes, GB) + + +def test_create_overlay_targets_the_instance_namespace(sample_hardware, sample_instances): + current = GlobalDeviceTree(hardware=sample_hardware, instances={}, device_references={}) + modified = GlobalDeviceTree( + hardware=sample_hardware, + instances={"database": sample_instances["database"]}, + device_references={}, + ) + fdt, ov = _ov(OverlayGenerator().generate_overlay(current, modified)) + + assert fdt.getprop(fdt.path_offset("/fragment@0"), "target-path").as_str() == "/instances" + create = fdt.subnode_offset(ov, "instance-create") + assert fdt.getprop(create, "instance-name").as_str() == "database" + # The kernel places instance memory itself, so the request names no base. + resources = fdt.subnode_offset(create, "resources") + assert fdt.getprop(resources, "memory-base", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + assert fdt.getprop(resources, "memory-bytes").as_uint64() == \ + sample_instances["database"].resources.memory_bytes + for offset in _walk(fdt): + assert fdt.getprop(offset, "mk,instance", quiet=[libfdt.FDT_ERR_NOTFOUND]) == MISSING + + +def test_removal_overlay_targets_the_instance_namespace(): + fdt, ov = _ov(OverlayGenerator().generate_removal_overlay("database")) + + assert fdt.getprop(fdt.path_offset("/fragment@0"), "target-path").as_str() == "/instances" + remove = fdt.subnode_offset(ov, "instance-remove") + assert fdt.getprop(remove, "instance-name").as_str() == "database" diff --git a/tests/test_resources.py b/tests/test_resources.py index 09112d4..8faf825 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -24,7 +24,9 @@ get_allocated_memory_regions_from_iomem, get_memory_pool_from_iomem, get_pool_allocated_bytes, - find_available_memory_base, + get_pool_chunks_from_iomem, + get_busy_chunks_from_iomem, + chunk_containing, validate_cpu_allocation, validate_memory_allocation, find_next_instance_id, @@ -44,6 +46,19 @@ def test_get_available_cpus(self, sample_tree): # So 16-31 should be available assert available == set(range(16, 32)) + def test_get_available_cpus_trusts_the_kernel_free_list(self, sample_tree): + """A read-back lists no instances, so its free list is the only truth.""" + sample_tree.hardware.cpus.available_free = [20, 21, 22] + sample_tree.instances = {} + + assert get_available_cpus(sample_tree) == {20, 21, 22} + + def test_get_available_cpus_free_list_minus_tree_instances(self, sample_tree): + """When the tree does list instances, their CPUs are not free.""" + sample_tree.hardware.cpus.available_free = [4, 5, 20] + + assert get_available_cpus(sample_tree) == {20} + def test_get_allocated_cpus(self, sample_tree): """Test getting allocated CPUs.""" allocated = get_allocated_cpus(sample_tree) @@ -100,38 +115,25 @@ def test_get_allocated_memory_regions(self, sample_tree): assert 0x80000000 in bases # web-server assert 0x100000000 in bases # database - def test_find_available_memory_base_empty_pool(self, sample_hardware): - """Test finding memory base in empty pool.""" - from kerf.models import GlobalDeviceTree - - # Create tree with no instances - tree = GlobalDeviceTree(hardware=sample_hardware, instances={}, device_references={}) + def test_chunk_containing_finds_the_holding_chunk(self, sample_tree): + """A region inside a chunk resolves to that chunk.""" + chunk = chunk_containing(sample_tree, 0x100000000, 1024**3) - # Request 1GB - size = 1024**3 - base = find_available_memory_base(tree, size, use_iomem=False) + assert chunk is sample_tree.hardware.memory.regions[0] - # Should get start of pool (aligned) - assert base == sample_hardware.memory.memory_pool_base + def test_chunk_containing_rejects_a_region_spanning_two_chunks(self, sample_hardware): + """Host memory between two chunks is not part of the pool.""" + from kerf.models import GlobalDeviceTree, PoolMemoryRegion - def test_find_available_memory_base_with_allocations(self, sample_tree): - """Test finding memory base with existing allocations.""" - # Request 1GB after existing allocations - size = 1024**3 - base = find_available_memory_base(sample_tree, size, use_iomem=False) - - # Should find a gap or append at end - assert base is not None - assert base >= sample_tree.hardware.memory.memory_pool_base - - def test_find_available_memory_base_no_space(self, sample_tree): - """Test finding memory base when no space available.""" - # Request more memory than available in pool - size = 100 * 1024**3 # 100GB - way more than pool size - base = find_available_memory_base(sample_tree, size, use_iomem=False) + sample_hardware.memory.regions = [ + PoolMemoryRegion(base=0x100000000, size=1024**3, node=0), + PoolMemoryRegion(base=0x200000000, size=1024**3, node=1), + ] + tree = GlobalDeviceTree(hardware=sample_hardware, instances={}, device_references={}) - # Should return None - assert base is None + assert chunk_containing(tree, 0x100000000, 1024**3) is not None + assert chunk_containing(tree, 0x200000000, 1024**3) is not None + assert chunk_containing(tree, 0x1C0000000, 1024**3) is None def test_validate_memory_allocation_success(self, sample_tree): """Test successful memory allocation validation.""" @@ -153,12 +155,12 @@ def test_validate_memory_allocation_overlap(self, sample_tree): validate_memory_allocation(sample_tree, memory_base, memory_bytes) def test_validate_memory_allocation_out_of_pool(self, sample_tree): - """Test memory allocation outside pool.""" - # Use base before pool - memory_base = 0x10000000 # Below pool base + """Test memory allocation outside every pool chunk.""" + # Use base before the only chunk + memory_base = 0x10000000 memory_bytes = 1024**3 - with pytest.raises(ResourceError, match="below pool base"): + with pytest.raises(ResourceError, match="does not fit in any pool chunk"): validate_memory_allocation(sample_tree, memory_base, memory_bytes) def test_validate_memory_allocation_misaligned(self, sample_tree): @@ -303,3 +305,38 @@ def test_get_pool_allocated_bytes_merges_nested_regions(self, tmp_path): ) usage = get_pool_allocated_bytes(str(path)) assert usage == (0x40000000, 0x40000000, 0x8000000) + + TWO_CHUNKS = "\n".join( + [ + "100000000-13fffffff : Multikernel Memory Pool", + " 100000000-10fffffff : mk-instance-1-web-region-0", + "200000000-21fffffff : Multikernel Memory Pool", + ] + ) + + @pytest.fixture + def two_chunk_iomem(self, tmp_path): + path = tmp_path / "iomem" + path.write_text(self.TWO_CHUNKS + "\n", encoding="utf-8") + return str(path) + + def test_pool_chunks_and_busy(self, two_chunk_iomem): + assert get_pool_chunks_from_iomem(two_chunk_iomem) == [ + (0x100000000, 1 << 30), + (0x200000000, 1 << 29), + ] + assert get_busy_chunks_from_iomem(two_chunk_iomem) == {0x100000000} + + def test_first_chunk_and_allocation_across_chunks(self, two_chunk_iomem): + assert get_memory_pool_from_iomem(two_chunk_iomem) == (0x100000000, 1 << 30) + assert get_pool_allocated_bytes(two_chunk_iomem) == ( + 0x100000000, + (1 << 30) + (1 << 29), + 1 << 28, + ) + + def test_no_chunks(self, tmp_path): + path = tmp_path / "iomem" + path.write_text("00001000-0009ffff : System RAM\n", encoding="utf-8") + assert get_pool_chunks_from_iomem(str(path)) == [] + assert get_busy_chunks_from_iomem(str(path)) == set() diff --git a/tests/test_runtime.py b/tests/test_runtime.py index e759dc0..e896e9a 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -184,7 +184,7 @@ def test_lock_acquisition(self): assert not manager.lock_file.exists() # Acquire lock - with manager._acquire_lock(): # pylint: disable=protected-access + with manager.lock(): # Lock should exist assert manager.lock_file.exists() @@ -203,7 +203,7 @@ def test_lock_timeout(self): try: # Try to acquire lock (should timeout) with pytest.raises(KernelInterfaceError, match="Could not acquire lock"): - with manager._acquire_lock(): # pylint: disable=protected-access + with manager.lock(): pass finally: # Cleanup diff --git a/tests/test_show.py b/tests/test_show.py new file mode 100644 index 0000000..05e4fd4 --- /dev/null +++ b/tests/test_show.py @@ -0,0 +1,160 @@ +# Copyright 2026 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Tests for `kerf show` baseline hardware display. +""" + +from kerf.models import ( + CPUAllocation, + GlobalDeviceTree, + HardwareInventory, + MemoryAllocation, + PoolMemoryRegion, +) +from kerf.show.main import display_baseline_info + + +def _tree(cpus, memory): + hardware = HardwareInventory(cpus=cpus, memory=memory, devices={}) + return GlobalDeviceTree(hardware=hardware, instances={}, device_references={}) + + +def _cpus(available_free=None): + return CPUAllocation( + total=32, + host_reserved=[0, 1, 2, 3], + available=list(range(4, 32)), + available_free=available_free, + ) + + +def test_lists_every_pool_chunk_with_its_node(capsys, monkeypatch): + memory = MemoryAllocation( + total_bytes=16 * 1024**3, + host_reserved_bytes=2 * 1024**3, + regions=[ + PoolMemoryRegion(base=0x100000000, size=4 * 1024**3, node=0), + PoolMemoryRegion(base=0x200000000, size=6 * 1024**3, node=1), + PoolMemoryRegion(base=0x300000000, size=1 * 1024**3, node=-1), + ], + ) + monkeypatch.setattr( + "kerf.show.main.get_pool_allocated_bytes", + lambda: (0x100000000, 11 * 1024**3, 3 * 1024**3), + ) + + display_baseline_info(_tree(_cpus(), memory)) + out = capsys.readouterr().out + + assert "Chunk: 0x100000000 4.00 GB node 0" in out + assert "Chunk: 0x200000000 6.00 GB node 1" in out + assert "Chunk: 0x300000000 1.00 GB" in out + assert "0x300000000 1.00 GB node" not in out + assert "Pool Allocated: 3.00 GB" in out + assert "Pool Available: 8.00 GB" in out + + +def test_no_allocation_summary_without_iomem(capsys, monkeypatch): + memory = MemoryAllocation( + total_bytes=0, + host_reserved_bytes=0, + regions=[PoolMemoryRegion(base=0x100000000, size=4 * 1024**3, node=0)], + ) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(_cpus(), memory)) + out = capsys.readouterr().out + + assert "Pool Allocated" not in out + assert "Pool Available" not in out + + +def test_memory_total_and_host_reserved_hidden_when_zero(capsys, monkeypatch): + memory = MemoryAllocation( + total_bytes=0, + host_reserved_bytes=0, + regions=[PoolMemoryRegion(base=0x100000000, size=4 * 1024**3, node=0)], + ) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(_cpus(), memory)) + out = capsys.readouterr().out + + # The CPU section prints its own Total and Host Reserved lines; only + # the memory ones (GB-suffixed) are suppressed when the value is 0. + assert "GB) " not in out + for line in out.splitlines(): + assert not line.strip().startswith("Total:") or "GB" not in line + assert not line.strip().startswith("Host Reserved:") or "GB" not in line + + +def test_memory_total_and_host_reserved_shown_when_nonzero(capsys, monkeypatch): + memory = MemoryAllocation( + total_bytes=16 * 1024**3, + host_reserved_bytes=2 * 1024**3, + regions=[PoolMemoryRegion(base=0x100000000, size=4 * 1024**3, node=0)], + ) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(_cpus(), memory)) + out = capsys.readouterr().out + + assert "Total: 16.00 GB" in out + assert "Host Reserved: 2.00 GB" in out + + +def test_cpu_pool_and_available_split_when_free_subset_known(capsys, monkeypatch): + memory = MemoryAllocation(total_bytes=0, host_reserved_bytes=0, regions=[]) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(_cpus(available_free=list(range(4, 20))), memory)) + out = capsys.readouterr().out + + assert "Pool CPUs: 28 cpus" in out + assert "Available CPUs: 16 cpus" in out + + +def test_cpu_available_line_kept_when_free_subset_unknown(capsys, monkeypatch): + memory = MemoryAllocation(total_bytes=0, host_reserved_bytes=0, regions=[]) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(_cpus(available_free=None), memory)) + out = capsys.readouterr().out + + assert "Available: 28 cpus" in out + assert "Pool CPUs" not in out + assert "Available CPUs" not in out + + +def test_pool_without_chunks_says_so(capsys, monkeypatch): + memory = MemoryAllocation(total_bytes=0, host_reserved_bytes=0, regions=[]) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(_cpus(), memory)) + out = capsys.readouterr().out + + assert "No memory pool configured" in out + assert "Chunk:" not in out + + +def test_cpu_host_reserved_hidden_when_the_kernel_reports_none(capsys, monkeypatch): + # A read-back describes the pool, not the host, and lists no reserved CPU. + cpus = CPUAllocation(total=32, host_reserved=[], available=list(range(4, 32))) + memory = MemoryAllocation(total_bytes=0, host_reserved_bytes=0, regions=[]) + monkeypatch.setattr("kerf.show.main.get_pool_allocated_bytes", lambda: None) + + display_baseline_info(_tree(cpus, memory)) + + assert "Host Reserved:" not in capsys.readouterr().out diff --git a/tests/test_topology.py b/tests/test_topology.py new file mode 100644 index 0000000..413d8fd --- /dev/null +++ b/tests/test_topology.py @@ -0,0 +1,106 @@ +# Copyright 2025 Multikernel Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reading the host NUMA topology out of sysfs and /proc/cpuinfo.""" + +from kerf.topology import cpu_numa_nodes, logical_to_apic, node_for_cpus + + +def _cpuinfo(tmp_path, apic_of_cpu): + path = tmp_path / "cpuinfo" + blocks = [] + for cpu, apic in sorted(apic_of_cpu.items()): + blocks.append( + f"processor\t: {cpu}\n" + "vendor_id\t: GenuineIntel\n" + "cpu family\t: 25\n" + f"apicid\t\t: {apic}\n" + f"initial apicid\t: {apic}\n" + ) + path.write_text("\n".join(blocks), encoding="utf-8") + return str(path) + + +def _node_root(tmp_path, cpulists): + root = tmp_path / "node" + root.mkdir() + for node, cpulist in cpulists.items(): + node_dir = root / f"node{node}" + node_dir.mkdir() + (node_dir / "cpulist").write_text(cpulist + "\n", encoding="utf-8") + return str(root) + + +def test_logical_cpus_map_to_apic_ids(tmp_path): + cpuinfo = _cpuinfo(tmp_path, {0: 0, 1: 2, 2: 4}) + + assert logical_to_apic(cpuinfo) == {0: 0, 1: 2, 2: 4} + + +def test_single_node_places_every_cpu(tmp_path): + cpuinfo = _cpuinfo(tmp_path, {0: 0, 1: 2, 2: 4, 3: 6}) + node_root = _node_root(tmp_path, {0: "0-3"}) + + mapping = cpu_numa_nodes(node_root, cpuinfo) + + assert mapping == {0: 0, 2: 0, 4: 0, 6: 0} + assert node_for_cpus([2, 4], mapping) == 0 + + +def test_two_nodes_keep_their_own_cpus(tmp_path): + cpuinfo = _cpuinfo(tmp_path, {0: 0, 1: 2, 2: 4, 3: 6}) + node_root = _node_root(tmp_path, {0: "0,1", 1: "2-3"}) + + mapping = cpu_numa_nodes(node_root, cpuinfo) + + assert mapping == {0: 0, 2: 0, 4: 1, 6: 1} + assert node_for_cpus([4, 6], mapping) == 1 + + +def test_cpus_split_across_nodes_follow_the_lowest_apic_id(tmp_path): + cpuinfo = _cpuinfo(tmp_path, {0: 0, 1: 2, 2: 4, 3: 6}) + node_root = _node_root(tmp_path, {0: "0,1", 1: "2-3"}) + + mapping = cpu_numa_nodes(node_root, cpuinfo) + + assert node_for_cpus([2, 4], mapping) == 0 + assert node_for_cpus([4, 2], mapping) == 0 + + +def test_offline_cpus_are_absent_from_the_mapping(tmp_path): + # The pool's own CPUs leave /proc/cpuinfo and the node cpulist. + cpuinfo = _cpuinfo(tmp_path, {0: 0}) + node_root = _node_root(tmp_path, {0: "0"}) + + mapping = cpu_numa_nodes(node_root, cpuinfo) + + assert node_for_cpus([2, 4], mapping) is None + + +def test_missing_files_leave_the_node_undecided(tmp_path): + missing = str(tmp_path / "nowhere") + + assert not cpu_numa_nodes(missing, missing) + assert node_for_cpus([0, 1], {}) is None + + cpuinfo = _cpuinfo(tmp_path, {0: 0}) + assert not cpu_numa_nodes(missing, cpuinfo) + assert not cpu_numa_nodes(_node_root(tmp_path, {0: "0"}), missing) + + +def test_memoryless_node_with_no_cpus_is_skipped(tmp_path): + cpuinfo = _cpuinfo(tmp_path, {0: 0, 1: 2}) + node_root = _node_root(tmp_path, {0: "0-1", 1: ""}) + + assert cpu_numa_nodes(node_root, cpuinfo) == {0: 0, 2: 0} diff --git a/tests/test_validator.py b/tests/test_validator.py index 76c9130..a6df621 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -16,6 +16,7 @@ Tests for kerf validator. """ +from kerf.dtc import validator as validator_module from kerf.dtc.validator import MultikernelValidator @@ -386,3 +387,90 @@ def test_validate_spread_affinity(self, sample_hardware): assert any( "Spread" in warning and "single NUMA node" in warning for warning in result.warnings ) + + +class TestPoolChunks: + """Test the pool read-back against /proc/iomem.""" + + @staticmethod + def _two_chunk_tree(sample_hardware): + from kerf.models import GlobalDeviceTree, PoolMemoryRegion + + sample_hardware.memory.regions = [ + PoolMemoryRegion(base=0x100000000, size=1024**3, node=0), + PoolMemoryRegion(base=0x200000000, size=512 * 1024**2, node=1), + ] + return GlobalDeviceTree(hardware=sample_hardware, instances={}, device_references={}) + + def test_every_chunk_registered_in_iomem_passes(self, sample_hardware, monkeypatch): + """A pool of several non-contiguous chunks is valid.""" + tree = self._two_chunk_tree(sample_hardware) + monkeypatch.setattr( + validator_module, "get_pool_chunks_from_iomem", + lambda: [(0x100000000, 1024**3), (0x200000000, 512 * 1024**2)], + ) + + result = MultikernelValidator().validate(tree) + + assert result.is_valid, result.errors + + def test_chunk_missing_from_iomem_fails(self, sample_hardware, monkeypatch): + """A chunk the kernel reports but /proc/iomem does not is an error.""" + tree = self._two_chunk_tree(sample_hardware) + monkeypatch.setattr( + validator_module, "get_pool_chunks_from_iomem", + lambda: [(0x100000000, 1024**3)], + ) + + result = MultikernelValidator().validate(tree) + + assert not result.is_valid + assert any("0x200000000" in error and "not registered in /proc/iomem" in error + for error in result.errors) + + def test_instance_inside_the_second_chunk_is_valid(self, sample_hardware, monkeypatch): + """An instance placed in a later chunk is not out of pool.""" + from kerf.models import Instance, InstanceResources + + tree = self._two_chunk_tree(sample_hardware) + tree.instances = { + "app1": Instance( + name="app1", + id=1, + resources=InstanceResources( + cpus=[4], memory_base=0x200000000, memory_bytes=128 * 1024**2, devices=[] + ), + ) + } + monkeypatch.setattr( + validator_module, "get_pool_chunks_from_iomem", + lambda: [(0x100000000, 1024**3), (0x200000000, 512 * 1024**2)], + ) + + result = MultikernelValidator().validate(tree) + + assert result.is_valid, result.errors + + def test_instance_spanning_two_chunks_fails(self, sample_hardware, monkeypatch): + """A region crossing the gap between chunks covers host memory.""" + from kerf.models import Instance, InstanceResources + + tree = self._two_chunk_tree(sample_hardware) + tree.instances = { + "app1": Instance( + name="app1", + id=1, + resources=InstanceResources( + cpus=[4], memory_base=0x1C0000000, memory_bytes=2 * 1024**3, devices=[] + ), + ) + } + monkeypatch.setattr( + validator_module, "get_pool_chunks_from_iomem", + lambda: [(0x100000000, 1024**3), (0x200000000, 512 * 1024**2)], + ) + + result = MultikernelValidator().validate(tree) + + assert not result.is_valid + assert any("does not fit in any pool chunk" in error for error in result.errors)