From c6d785868b0b5f1617ba9bc33f7fdd89e94a21a7 Mon Sep 17 00:00:00 2001 From: Anders Roxell Date: Fri, 5 Dec 2025 16:07:36 +0100 Subject: [PATCH 1/5] config: docker: kernelci: Add tuxmake Add tuxmake kernel build tool to the kernelci Docker fragment. Pin to version 1.35.0 for reproducibility. Reviewed-by: Ben Copeland Signed-off-by: Anders Roxell --- config/docker/fragment/kernelci.jinja2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/docker/fragment/kernelci.jinja2 b/config/docker/fragment/kernelci.jinja2 index d5c552b3a2..9394ed45e4 100644 --- a/config/docker/fragment/kernelci.jinja2 +++ b/config/docker/fragment/kernelci.jinja2 @@ -18,6 +18,9 @@ RUN cp -R config /etc/kernelci/ WORKDIR /root RUN rm -rf /tmp/kernelci-core +# Install tuxmake kernel build tool +RUN pip3 install --break-system-packages --no-cache-dir tuxmake==1.35.0 + # Set up kernelci user RUN useradd kernelci -u 1000 -d /home/kernelci -s /bin/bash RUN mkdir -p /home/kernelci From 2d077790a12b62c2dd5bb66048b8539b97674050 Mon Sep 17 00:00:00 2001 From: Anders Roxell Date: Tue, 9 Dec 2025 09:14:32 +0100 Subject: [PATCH 2/5] kbuild: Add backend parameter and refactor build logic Add a 'backend' parameter that defaults to 'make' for backward compatibility. Extract the make build logic from _generate_script() into a dedicated _build_with_make() method so different backends can have their own build implementations. Reviewed-by: Ben Copeland Signed-off-by: Anders Roxell --- kernelci/kbuild.py | 54 ++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/kernelci/kbuild.py b/kernelci/kbuild.py index 0dacb8c683..022755b32c 100644 --- a/kernelci/kbuild.py +++ b/kernelci/kbuild.py @@ -165,6 +165,7 @@ def __init__(self, node=None, jobname=None, params=None, jsonobj=None, apiconfig # if defconfig contains '+', it means it is a list if isinstance(self._defconfig, str) and '+' in self._defconfig: self._defconfig = self._defconfig.split('+') + self._backend = params.get('backend', 'make') self._fragments = params['fragments'] self._fragment_configs = fragment_configs or {} if 'coverage' in self._fragments: @@ -220,6 +221,7 @@ def __init__(self, node=None, jobname=None, params=None, jsonobj=None, apiconfig self._defconfig = jsonobj['defconfig'] self._fragments = jsonobj['fragments'] self._fragment_configs = jsonobj.get('fragment_configs', {}) + self._backend = jsonobj.get('backend', 'make') self._cross_compile = jsonobj['cross_compile'] self._cross_compile_compat = jsonobj['cross_compile_compat'] self._steps = jsonobj['steps'] @@ -635,30 +637,10 @@ def _merge_frags(self, fragnum): def _generate_script(self): """ Generate shell script for complete build """ - # TODO(nuclearcat): Fetch firmware only if needed print("Generating shell script") fragnum = self._parse_fragments(firmware=True) self._merge_frags(fragnum) - if not self._dtbs_check: - # TODO: verify if CONFIG_EXTRA_FIRMWARE have any files - # We can check that if fragments have CONFIG_EXTRA_FIRMWARE - self._fetch_firmware() - self._build_kernel() - self._build_modules() - if self._kfselftest: - self._build_kselftest() - if self._arch not in DTBS_DISABLED: - self._build_dtbs() - self._package_kimage() - self._package_modules() - if self._coverage: - self._package_coverage() - if self._kfselftest: - self._package_kselftest() - if self._arch not in DTBS_DISABLED: - self._package_dtbs() - else: - self._build_dtbs_check() + self._build_with_make() self._write_metadata() # terminate all active jobs self.startjob(None) @@ -697,6 +679,27 @@ def write_script(self, filename): # copy to artifacts dir os.system(f"cp {filename} {self._af_dir}/build.sh") + def _build_with_make(self): + """ Build kernel using make """ + if not self._dtbs_check: + self._fetch_firmware() + self._build_kernel() + self._build_modules() + if self._kfselftest: + self._build_kselftest() + if self._arch not in DTBS_DISABLED: + self._build_dtbs() + self._package_kimage() + self._package_modules() + if self._coverage: + self._package_coverage() + if self._kfselftest: + self._package_kselftest() + if self._arch not in DTBS_DISABLED: + self._package_dtbs() + else: + self._build_dtbs_check() + def _build_kernel(self): """ Add kernel build steps """ self.startjob("build_kernel") @@ -859,13 +862,18 @@ def _write_metadata(self): metadata['build']['fragments'] = self._fragments metadata['build']['srcdir'] = self._srcdir metadata['build']['config_full'] = self._config_full + metadata['build']['backend'] = self._backend with open(os.path.join(self._af_dir, "metadata.json"), 'w') as f: json.dump(metadata, f, indent=4) def serialize(self, filename): - """ Serialize class to json """ - # TODO(nuclearcat): Implement to_json method? + """ Serialize class to json + + Note: Uses __dict__ to serialize all instance attributes (including + _backend, _arch, etc). The from_json() method strips underscore + prefixes when loading, so _backend becomes 'backend' in jsonobj. + """ data = json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4) with open(filename, 'w') as f: From f0372598dabc74b0c4cd273faa1fd00ed1c8d0d3 Mon Sep 17 00:00:00 2001 From: Anders Roxell Date: Tue, 9 Dec 2025 09:20:32 +0100 Subject: [PATCH 3/5] kbuild: Add tuxmake backend support Add tuxmake as an alternative build backend, enabled with the USE_TUXMAKE=1 environment variable. Refactor _parse_fragments() to return a list of fragment file paths instead of a count and update _merge_frags() to accept this list. Add _build_with_tuxmake() method with basic tuxmake invocation. Only call _merge_frags() for the make backend since tuxmake handles fragments via --kconfig-add, added in the next commit. Reviewed-by: Ben Copeland Signed-off-by: Anders Roxell --- kernelci/kbuild.py | 143 ++++++++++++++++++++++++++++++--------------- 1 file changed, 96 insertions(+), 47 deletions(-) diff --git a/kernelci/kbuild.py b/kernelci/kbuild.py index 022755b32c..b55f2407f5 100644 --- a/kernelci/kbuild.py +++ b/kernelci/kbuild.py @@ -166,6 +166,9 @@ def __init__(self, node=None, jobname=None, params=None, jsonobj=None, apiconfig if isinstance(self._defconfig, str) and '+' in self._defconfig: self._defconfig = self._defconfig.split('+') self._backend = params.get('backend', 'make') + # Support USE_TUXMAKE environment variable for backward compatibility + if os.environ.get('USE_TUXMAKE') == '1': + self._backend = 'tuxmake' self._fragments = params['fragments'] self._fragment_configs = fragment_configs or {} if 'coverage' in self._fragments: @@ -313,20 +316,21 @@ def init_steps(self): # set environment variables self.addcomment("Set environment variables") self.addcmd("export ARCH=" + self._arch) - if self._cross_compile: - self.addcmd("export CROSS_COMPILE=" + self._cross_compile) - if self._cross_compile_compat: - self.addcmd("export CROSS_COMPILE_COMPAT=" + - self._cross_compile_compat) - self.addcmd("export INSTALL_MOD_PATH=_modules_") - self.addcmd("export INSTALL_MOD_STRIP=1") - self.addcmd("export INSTALL_DTBS_PATH=_dtbs_") - self.addcmd("export CC=" + self._compiler) - self.addcmd("export HOSTCC=" + self._compiler) - # if self._compiler start with clang- we need to set env vars - if self._compiler.startswith("clang-"): - # LLVM=1, can be suffix with version in future, like -14 - self.addcmd("export LLVM=1") + if self._backend != 'tuxmake': + if self._cross_compile: + self.addcmd("export CROSS_COMPILE=" + self._cross_compile) + if self._cross_compile_compat: + self.addcmd("export CROSS_COMPILE_COMPAT=" + + self._cross_compile_compat) + self.addcmd("export INSTALL_MOD_PATH=_modules_") + self.addcmd("export INSTALL_MOD_STRIP=1") + self.addcmd("export INSTALL_DTBS_PATH=_dtbs_") + self.addcmd("export CC=" + self._compiler) + self.addcmd("export HOSTCC=" + self._compiler) + # if self._compiler start with clang- we need to set env vars + if self._compiler.startswith("clang-"): + # LLVM=1, can be suffix with version in future, like -14 + self.addcmd("export LLVM=1") # set -x for echo self._steps.append("set -x") # touch build.log @@ -339,21 +343,22 @@ def init_steps(self): # but keep pid, so i can kill it later self._artifacts.append("build.log") self._artifacts.append("build.sh") - if not self._dtbs_check: - self._artifacts.append("build_kimage.log") - self._artifacts.append("build_kimage_stderr.log") - self._artifacts.append("build_modules.log") - self._artifacts.append("build_modules_stderr.log") - if self._kfselftest: - self._artifacts.append("build_kselftest.log") - self._artifacts.append("build_kselftest_stderr.log") - # disable DTBS for some archs - if self._arch not in DTBS_DISABLED: - self._artifacts.append("build_dtbs.log") - self._artifacts.append("build_dtbs_stderr.log") - else: - self._artifacts.append("build_dtbs_check.log") - self._artifacts.append("build_dtbs_check_stderr.log") + if self._backend != 'tuxmake': + if not self._dtbs_check: + self._artifacts.append("build_kimage.log") + self._artifacts.append("build_kimage_stderr.log") + self._artifacts.append("build_modules.log") + self._artifacts.append("build_modules_stderr.log") + if self._kfselftest: + self._artifacts.append("build_kselftest.log") + self._artifacts.append("build_kselftest_stderr.log") + # disable DTBS for some archs + if self._arch not in DTBS_DISABLED: + self._artifacts.append("build_dtbs.log") + self._artifacts.append("build_dtbs_stderr.log") + else: + self._artifacts.append("build_dtbs_check.log") + self._artifacts.append("build_dtbs_check_stderr.log") self._artifacts.append("metadata.json") # download tarball self.addcomment("Download tarball") @@ -567,12 +572,19 @@ def add_fragment(self, fragname): return self.extract_config(frag) def _parse_fragments(self, firmware=False): - """ Parse fragments kbuild config and create config fragments """ - num = 0 - for fragment in self._fragments: + """ Parse fragments kbuild config and create config fragments + + Returns: + list: List of fragment file paths + """ + fragment_files = [] + + for idx, fragment in enumerate(self._fragments): content = '' + fragment_name = fragment + if fragment.startswith("cros://"): - (content, fragment) = self._getcrosfragment(fragment) + (content, fragment_name) = self._getcrosfragment(fragment) elif fragment.startswith("cip://"): content = self._getcipfragment(fragment) elif fragment.startswith("CONFIG_"): @@ -581,27 +593,37 @@ def _parse_fragments(self, firmware=False): # Use fragment configs passed from scheduler content = self.add_fragment(fragment) - fragfile = os.path.join(self._fragments_dir, f"{num}.config") + fragfile = os.path.join(self._fragments_dir, f"{idx}.config") with open(fragfile, 'w') as f: f.write(content) + + fragment_files.append(fragfile) + # add fragment to artifacts but relative to artifacts dir frag_rel = os.path.relpath(fragfile, self._af_dir) - self._config_full += '+' + fragment + self._config_full += '+' + fragment_name self._artifacts.append(frag_rel) - num += 1 + if firmware: content = 'CONFIG_EXTRA_FIRMWARE_DIR="'+self._firmware_dir+'"\n' - fragfile = os.path.join(self._fragments_dir, f"{num}.config") + fragfile = os.path.join(self._fragments_dir, f"{len(self._fragments)}.config") with open(fragfile, 'w') as f: f.write(content) + + fragment_files.append(fragfile) + # add fragment to artifacts but relative to artifacts dir frag_rel = os.path.relpath(fragfile, self._af_dir) self._artifacts.append(frag_rel) - num += 1 - return num - def _merge_frags(self, fragnum): - """ Merge config fragments to .config """ + return fragment_files + + def _merge_frags(self, fragment_files): + """ Merge config fragments to .config + + Args: + fragment_files: List of fragment file paths to merge + """ self.startjob("config_defconfig") self.addcmd("cd " + self._srcdir) if isinstance(self._defconfig, str) and self._defconfig.startswith('cros://'): @@ -625,9 +647,8 @@ def _merge_frags(self, fragnum): self._config_full = defconfigs + self._config_full # fragments self.startjob("config_fragments") - for i in range(0, fragnum): - self.addcmd("./scripts/kconfig/merge_config.sh" + - f" -m .config {self._fragments_dir}/{i}.config") + for fragfile in fragment_files: + self.addcmd(f"./scripts/kconfig/merge_config.sh -m .config {fragfile}") # TODO: olddefconfig should be optional/configurable # TODO: log all warnings/errors of olddefconfig to separate file self.addcmd("make olddefconfig") @@ -638,9 +659,14 @@ def _merge_frags(self, fragnum): def _generate_script(self): """ Generate shell script for complete build """ print("Generating shell script") - fragnum = self._parse_fragments(firmware=True) - self._merge_frags(fragnum) - self._build_with_make() + self._fragment_files = self._parse_fragments(firmware=True) + + if self._backend == 'tuxmake': + self._build_with_tuxmake() + else: + self._merge_frags(self._fragment_files) + self._build_with_make() + self._write_metadata() # terminate all active jobs self.startjob(None) @@ -700,6 +726,29 @@ def _build_with_make(self): else: self._build_dtbs_check() + def _build_with_tuxmake(self): + """ Build kernel using tuxmake """ + self.startjob("build_tuxmake") + self.addcmd("cd " + self._srcdir) + + tuxmake_cmd = ( + f"tuxmake --runtime=null " + f"--target-arch={self._arch} " + f"--toolchain={self._compiler} " + f"kernel modules" + ) + + print(f"Building with tuxmake: {tuxmake_cmd}") + self.addcmd(tuxmake_cmd) + + # tuxmake outputs 'config' (no dot), rename to '.config' for consistency + self.addcmd( + f"[ -f {self._af_dir}/config ] && " + f"mv {self._af_dir}/config {self._af_dir}/.config" + ) + + self.addcmd("cd ..") + def _build_kernel(self): """ Add kernel build steps """ self.startjob("build_kernel") From 6f81823a7270dd4df163b6805ade93401d06f691 Mon Sep 17 00:00:00 2001 From: Anders Roxell Date: Tue, 9 Dec 2025 09:43:15 +0100 Subject: [PATCH 4/5] kbuild: Use tuxmake native fragment support with --kconfig-add Switch to tuxmake's native --kconfig-add parameter for applying configuration fragments instead of manually merging them. Add --output-dir to specify where tuxmake places build artifacts. Build the dtbs target for architectures that support it. Handle multiple defconfigs by passing the first to --kconfig and the rest as --kconfig-add. Handle ChromeOS defconfig by pre-creating the .config file. Reviewed-by: Ben Copeland Signed-off-by: Anders Roxell --- kernelci/kbuild.py | 100 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 89 insertions(+), 11 deletions(-) diff --git a/kernelci/kbuild.py b/kernelci/kbuild.py index b55f2407f5..b42817b408 100644 --- a/kernelci/kbuild.py +++ b/kernelci/kbuild.py @@ -593,10 +593,17 @@ def _parse_fragments(self, firmware=False): # Use fragment configs passed from scheduler content = self.add_fragment(fragment) + if not content: + print(f"[_parse_fragments] WARNING: Fragment {fragment} has no content") + continue + fragfile = os.path.join(self._fragments_dir, f"{idx}.config") with open(fragfile, 'w') as f: f.write(content) + config_count = len([line for line in content.split('\n') if line.strip()]) + print(f"[_parse_fragments] Created {fragfile} ({config_count} configs)") + fragment_files.append(fragfile) # add fragment to artifacts but relative to artifacts dir @@ -616,6 +623,7 @@ def _parse_fragments(self, firmware=False): frag_rel = os.path.relpath(fragfile, self._af_dir) self._artifacts.append(frag_rel) + print(f"[_parse_fragments] Created {len(fragment_files)} fragment files") return fragment_files def _merge_frags(self, fragment_files): @@ -727,18 +735,78 @@ def _build_with_make(self): self._build_dtbs_check() def _build_with_tuxmake(self): - """ Build kernel using tuxmake """ + """ Build kernel using tuxmake with native fragment support """ + print("[_build_with_tuxmake] Starting tuxmake build") + + if not hasattr(self, '_fragment_files'): + print("[_build_with_tuxmake] ERROR: No fragment files available") + self._fragment_files = [] + + print(f"[_build_with_tuxmake] Using {len(self._fragment_files)} fragment files") + + # Handle defconfigs - first goes to --kconfig, rest to --kconfig-add + extra_defconfigs = [] + if isinstance(self._defconfig, list): + defconfig = self._defconfig[0] + extra_defconfigs = self._defconfig[1:] + self._config_full = '+'.join(self._defconfig) + self._config_full + elif isinstance(self._defconfig, str): + defconfig = self._defconfig + self._config_full = self._defconfig + self._config_full + else: + defconfig = 'defconfig' + print("[_build_with_tuxmake] WARNING: No defconfig specified, using 'defconfig'") + + # Fetch firmware for builds that need it + self._fetch_firmware() + self.startjob("build_tuxmake") self.addcmd("cd " + self._srcdir) - tuxmake_cmd = ( - f"tuxmake --runtime=null " - f"--target-arch={self._arch} " - f"--toolchain={self._compiler} " - f"kernel modules" - ) + use_kconfig_flag = True - print(f"Building with tuxmake: {tuxmake_cmd}") + # Handle ChromeOS defconfig + if defconfig.startswith('cros://'): + print(f"[_build_with_tuxmake] Handling ChromeOS defconfig: {defconfig}") + dotconfig = os.path.join(self._srcdir, ".config") + content, defconfig_name = self._getcrosfragment(defconfig) + with open(dotconfig, 'w') as f: + f.write(content) + self.addcmd("make olddefconfig") + use_kconfig_flag = False + + cmd_parts = [ + "tuxmake --runtime=null", + f"--target-arch={self._arch}", + f"--toolchain={self._compiler}", + f"--output-dir={self._af_dir}", + ] + + if use_kconfig_flag: + cmd_parts.append(f"--kconfig={defconfig}") + for extra in extra_defconfigs: + cmd_parts.append(f"--kconfig-add={extra}") + print(f"[_build_with_tuxmake] Adding extra defconfig: {extra}") + + for fragfile in self._fragment_files: + if os.path.exists(fragfile): + cmd_parts.append(f"--kconfig-add={fragfile}") + print(f"[_build_with_tuxmake] Adding fragment: {os.path.basename(fragfile)}") + else: + print(f"[_build_with_tuxmake] WARNING: Fragment file not found: {fragfile}") + + # Build targets: kernel modules, plus dtbs if arch supports it + targets = ["kernel", "modules"] + if self._arch not in DTBS_DISABLED: + targets.append("dtbs") + if self._kfselftest: + targets.append("kselftest") + cmd_parts.append(" ".join(targets)) + print(f"[_build_with_tuxmake] Building targets: {' '.join(targets)}") + + tuxmake_cmd = " ".join(cmd_parts) + print(f"[_build_with_tuxmake] Command: {tuxmake_cmd}") + print(f"[_build_with_tuxmake] Output directory: {self._af_dir}") self.addcmd(tuxmake_cmd) # tuxmake outputs 'config' (no dot), rename to '.config' for consistency @@ -996,9 +1064,19 @@ def upload_artifacts(self): # Prepare all artifacts for upload upload_tasks = [] - for artifact in self._artifacts: - artifact_path = os.path.join(self._af_dir, artifact) - upload_tasks.append((artifact, artifact_path)) + if self._backend == 'tuxmake': + # For TuxMake, upload everything in artifacts directory + print("[_upload_artifacts] TuxMake backend: discovering files in artifacts dir") + for root, dirs, files in os.walk(self._af_dir): + for file in files: + file_rel = os.path.relpath(os.path.join(root, file), self._af_dir) + artifact_path = os.path.join(self._af_dir, file_rel) + upload_tasks.append((file_rel, artifact_path)) + else: + # For make backend, upload only listed artifacts + for artifact in self._artifacts: + artifact_path = os.path.join(self._af_dir, artifact) + upload_tasks.append((artifact, artifact_path)) # Function to handle a single artifact upload # args: (artifact, artifact_path) From 8793789cf3f8329e485dcdaac8885db05fdc3116 Mon Sep 17 00:00:00 2001 From: Anders Roxell Date: Thu, 8 Jan 2026 20:51:10 +0100 Subject: [PATCH 5/5] kbuild: backend tuxmake: Add dtbs_check support Add dtbs_check target support for the tuxmake backend. When dtbs_check is requested, run only the dtbs_check target instead of the normal kernel, modules and dtbs targets. Also skip firmware fetch for dtbs_check builds since it is not needed. Reviewed-by: Ben Copeland Signed-off-by: Anders Roxell --- kernelci/kbuild.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/kernelci/kbuild.py b/kernelci/kbuild.py index b42817b408..7ef5a844a5 100644 --- a/kernelci/kbuild.py +++ b/kernelci/kbuild.py @@ -757,8 +757,9 @@ def _build_with_tuxmake(self): defconfig = 'defconfig' print("[_build_with_tuxmake] WARNING: No defconfig specified, using 'defconfig'") - # Fetch firmware for builds that need it - self._fetch_firmware() + # Fetch firmware only for normal builds, not dtbs_check + if not self._dtbs_check: + self._fetch_firmware() self.startjob("build_tuxmake") self.addcmd("cd " + self._srcdir) @@ -795,10 +796,14 @@ def _build_with_tuxmake(self): else: print(f"[_build_with_tuxmake] WARNING: Fragment file not found: {fragfile}") - # Build targets: kernel modules, plus dtbs if arch supports it - targets = ["kernel", "modules"] - if self._arch not in DTBS_DISABLED: - targets.append("dtbs") + # Build targets depend on mode + if self._dtbs_check: + targets = ["dtbs_check"] + else: + # Normal build: kernel, modules, plus dtbs if arch supports it + targets = ["kernel", "modules"] + if self._arch not in DTBS_DISABLED: + targets.append("dtbs") if self._kfselftest: targets.append("kselftest") cmd_parts.append(" ".join(targets))