From d7211ed88846327e94c023ef80b7bbead9d54736 Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Thu, 17 Sep 2026 15:20:59 +0100 Subject: [PATCH 01/14] Add support for newer RTools releases --- cmdstanpy/install_cmdstan.py | 46 ++-- cmdstanpy/install_cxx_toolchain.py | 164 ++++++++----- cmdstanpy/utils/__init__.py | 12 + cmdstanpy/utils/cmdstan.py | 359 ++++++++++++++++++----------- 4 files changed, 375 insertions(+), 206 deletions(-) diff --git a/cmdstanpy/install_cmdstan.py b/cmdstanpy/install_cmdstan.py index 06be41f9..638021d3 100644 --- a/cmdstanpy/install_cmdstan.py +++ b/cmdstanpy/install_cmdstan.py @@ -545,26 +545,47 @@ def retrieve_version(version: str, progress: bool = True) -> None: def run_compiler_install(dir: str, verbose: bool, progress: bool) -> None: from .install_cxx_toolchain import is_installed as _is_installed_cxx + from .install_cxx_toolchain import latest_version as _latest_version_cxx from .install_cxx_toolchain import run_rtools_install as _main_cxx - from .utils import cxx_toolchain_path + from .utils import cxx_toolchain_path, determine_windows_arch + + arch = determine_windows_arch() + known_versions = ['4.5', '4.4', '4.3', '4.2', '4.0', '3.5'] compiler_found = False - rtools40_home = os.environ.get('RTOOLS40_HOME') - for cxx_loc in ([rtools40_home] if rtools40_home is not None else []) + [ - home_cmdstan(), - os.path.join(os.path.abspath("/"), "RTools40"), - os.path.join(os.path.abspath("/"), "RTools"), - os.path.join(os.path.abspath("/"), "RTools35"), - os.path.join(os.path.abspath("/"), "RBuildTools"), - ]: - for cxx_version in ['40', '35']: - if _is_installed_cxx(cxx_loc, cxx_version): + cxx_version = _latest_version_cxx() + homes = [ + home + for home in ( + os.environ.get(var) + for var in ( + 'RTOOLS45_HOME', + 'RTOOLS44_HOME', + 'RTOOLS43_HOME', + 'RTOOLS42_HOME', + 'RTOOLS40_HOME', + ) + ) + if home + ] + names = ['RTools45', 'RTools44', 'RTools40', 'RTools35', 'RTools'] + if arch == 'aarch64': + names = ['RTools45-aarch64', 'RTools44-aarch64'] + names + for cxx_loc in ( + homes + + [home_cmdstan()] + + [os.path.join(os.path.abspath("/"), name) for name in names] + + [os.path.join(os.path.abspath("/"), "RBuildTools")] + ): + for version in known_versions: + if _is_installed_cxx(cxx_loc, version): + cxx_version = version compiler_found = True break if compiler_found: break if not compiler_found: - print('Installing RTools40') + print(f'Installing RTools {cxx_version}') # copy argv and clear sys.argv _main_cxx( { @@ -574,7 +595,6 @@ def run_compiler_install(dir: str, verbose: bool, progress: bool) -> None: 'verbose': verbose, } ) - cxx_version = '40' # Add toolchain to $PATH cxx_toolchain_path(cxx_version, dir) diff --git a/cmdstanpy/install_cxx_toolchain.py b/cmdstanpy/install_cxx_toolchain.py index 4fbf86b9..68b2c417 100644 --- a/cmdstanpy/install_cxx_toolchain.py +++ b/cmdstanpy/install_cxx_toolchain.py @@ -2,7 +2,7 @@ """ Download and install a C++ toolchain. Currently implemented platforms (platform.system) - Windows: RTools 3.5, 4.0 (default) + Windows: RTools 3.5, 4.0 (default on x86), 4.4, 4.5 (default on ARM64) Darwin (macOS): Not implemented Linux: Not implemented Optional command line arguments: @@ -25,17 +25,38 @@ from typing import Any from cmdstanpy import _DOT_CMDSTAN -from cmdstanpy.utils import pushd, validate_dir, wrap_url_progress_hook +from cmdstanpy.utils import ( + determine_windows_arch, + normalize_rtools_version, + pushd, + rtools_compiler, + rtools_layouts, + validate_dir, + wrap_url_progress_hook, +) EXTENSION = '.exe' if platform.system() == 'Windows' else '' -IS_64BITS = sys.maxsize > 2**32 + +# RTools 4.2 and later embed toolchain and installer build revisions in the +# installer filename, so there is no stable URL. Refresh when CRAN publishes +# a new build; see https://cran.r-project.org/bin/windows/Rtools/ +RTOOLS_INSTALLERS = { + '4.5': { + 'x86_64': 'rtools45-6768-6492.exe', + 'aarch64': 'rtools45-aarch64-6768-6492.exe', + }, + '4.4': { + 'x86_64': 'rtools44-6459-6401.exe', + 'aarch64': 'rtools44-aarch64-6459-6401.exe', + }, +} def usage() -> None: """Print usage.""" print( """Arguments: - -v (--version) :CmdStan version + -v (--version) : RTools version: 3.5, 4.0, 4.4 or 4.5 -d (--dir) : install directory -s (--silent) : install with /VERYSILENT instead of /SILENT for RTools -m (--no-make) : don't install mingw32-make (Windows RTools 4.0 only) @@ -111,13 +132,14 @@ def install_version( def install_mingw32_make(toolchain_loc: str, verbose: bool = False) -> None: """Install mingw32-make for Windows RTools 4.0.""" + arch = determine_windows_arch() os.environ['PATH'] = ';'.join( list( OrderedDict.fromkeys( [ os.path.join( toolchain_loc, - 'mingw_64' if IS_64BITS else 'mingw_32', + 'mingw64' if arch == 'x86_64' else 'mingw32', 'bin', ), os.path.join(toolchain_loc, 'usr', 'bin'), @@ -129,7 +151,11 @@ def install_mingw32_make(toolchain_loc: str, verbose: bool = False) -> None: cmd = [ 'pacman', '-Sy', - 'mingw-w64-x86_64-make' if IS_64BITS else 'mingw-w64-i686-make', + ( + 'mingw-w64-x86_64-make' + if arch == 'x86_64' + else 'mingw-w64-i686-make' + ), '--noconfirm', ] with pushd('.'): @@ -162,37 +188,25 @@ def install_mingw32_make(toolchain_loc: str, verbose: bool = False) -> None: def is_installed(toolchain_loc: str, version: str) -> bool: """Returns True is toolchain is installed.""" - if platform.system() == 'Windows': - if version in ['35', '3.5']: - if not os.path.exists(os.path.join(toolchain_loc, 'bin')): - return False - return os.path.exists( - os.path.join( - toolchain_loc, - 'mingw_64' if IS_64BITS else 'mingw_32', - 'bin', - 'g++' + EXTENSION, - ) - ) - elif version in ['40', '4.0', '4']: - return os.path.exists( - os.path.join( - toolchain_loc, - 'mingw64' if IS_64BITS else 'mingw32', - 'bin', - 'g++' + EXTENSION, - ) - ) - else: - return False + if platform.system() != 'Windows': + return False + for layout in rtools_layouts(normalize_rtools_version(version)): + tool_path = os.path.join(toolchain_loc, *layout.tool_subdir) + if not os.path.exists(tool_path): + continue + if rtools_compiler(toolchain_loc, layout) is not None: + return True return False def latest_version() -> str: - """Windows version hardcoded to 4.0.""" - if platform.system() == 'Windows': - return '4.0' - return '' + """Latest RTools version supported on this machine.""" + if platform.system() != 'Windows': + return '' + if determine_windows_arch() == 'aarch64': + # RTools 4.0 has no ARM64 build + return '4.5' + return '4.0' def retrieve_toolchain(filename: str, url: str, progress: bool = True) -> None: @@ -222,10 +236,7 @@ def retrieve_toolchain(filename: str, url: str, progress: bool = True) -> None: def normalize_version(version: str) -> str: """Return maj.min part of version string.""" if platform.system() == 'Windows': - if version in ['4', '40']: - version = '4.0' - elif version == '35': - version = '3.5' + return normalize_rtools_version(version) return version @@ -237,28 +248,44 @@ def get_toolchain_name() -> str: # TODO(2.0): consider something other than RTools -def get_url(version: str) -> str: +def get_url(version: str, arch: str | None = None) -> str: """Return URL for toolchain.""" - url = '' - if platform.system() == 'Windows': - if version == '4.0': - # pylint: disable=line-too-long - if IS_64BITS: - url = 'https://cran.r-project.org/bin/windows/Rtools/rtools40-x86_64.exe' # noqa: disable=E501 - else: - url = 'https://cran.r-project.org/bin/windows/Rtools/rtools40-i686.exe' # noqa: disable=E501 - elif version == '3.5': - url = 'https://cran.r-project.org/bin/windows/Rtools/Rtools35.exe' - return url - + if platform.system() != 'Windows': + return '' + if arch is None: + arch = determine_windows_arch() + if version in RTOOLS_INSTALLERS: + installer = RTOOLS_INSTALLERS[version].get(arch, '') + if not installer: + return '' + series = 'rtools' + version.replace('.', '') + return ( + 'https://cran.r-project.org/bin/windows/Rtools/' + f'{series}/files/{installer}' + ) + legacy = { + ('4.0', 'x86_64'): 'rtools40-x86_64.exe', + ('4.0', 'i686'): 'rtools40-i686.exe', + ('3.5', 'x86_64'): 'Rtools35.exe', + ('3.5', 'i686'): 'Rtools35.exe', + }.get((version, arch), '') + if legacy: + return f'https://cran.r-project.org/bin/windows/Rtools/{legacy}' + return '' -def get_toolchain_version(name: str, version: str) -> str: - """Toolchain version.""" - toolchain_folder = '' - if platform.system() == 'Windows': - toolchain_folder = '{}{}'.format(name, version.replace('.', '')) - return toolchain_folder +def get_toolchain_version( + name: str, version: str, arch: str | None = None +) -> str: + """Toolchain install folder name.""" + if platform.system() != 'Windows': + return '' + if arch is None: + arch = determine_windows_arch() + folder = '{}{}'.format(name, version.replace('.', '')) + if arch == 'aarch64': + folder += '-aarch64' + return folder def run_rtools_install(args: dict[str, Any]) -> None: @@ -274,9 +301,18 @@ def run_rtools_install(args: dict[str, Any]) -> None: if version is None: version = latest_version() version = normalize_version(version) - print("C++ toolchain '{}' version: {}".format(toolchain, version)) + arch = determine_windows_arch() + print( + "C++ toolchain '{}' version: {} ({})".format(toolchain, version, arch) + ) - url = get_url(version) + url = get_url(version, arch) + if not url: + raise ValueError( + f'RTools {version} is not available for {arch}. ' + f'Supported: {", ".join(sorted(RTOOLS_INSTALLERS))}, 4.0, 3.5 ' + '(4.0 and 3.5 are x86 only).' + ) if 'verbose' in args: verbose = args['verbose'] @@ -297,12 +333,12 @@ def run_rtools_install(args: dict[str, Any]) -> None: if platform.system() == 'Windows': silent = 'silent' in args # force silent == False for 4.0 version - if 'silent' not in args and version in ('4.0', '4', '40'): + if 'silent' not in args and version == '4.0': silent = False else: silent = False - toolchain_folder = get_toolchain_version(toolchain, version) + toolchain_folder = get_toolchain_version(toolchain, version, arch) with pushd(install_dir): if is_installed(toolchain_folder, version): print('C++ toolchain {} already installed'.format(toolchain_folder)) @@ -322,7 +358,7 @@ def run_rtools_install(args: dict[str, Any]) -> None: if ( 'no-make' not in args and (platform.system() == 'Windows') - and (version in ('4.0', '4', '40')) + and version == '4.0' ): if os.path.exists( os.path.join( @@ -336,7 +372,11 @@ def run_rtools_install(args: dict[str, Any]) -> None: def parse_cmdline_args() -> dict[str, Any]: parser = argparse.ArgumentParser() - parser.add_argument('--version', '-v', help="version, defaults to latest") + parser.add_argument( + '--version', + '-v', + help="RTools version (3.5, 4.0, 4.4, 4.5), defaults to latest", + ) parser.add_argument( '--dir', '-d', help="install directory, defaults to '~/.cmdstan" ) diff --git a/cmdstanpy/utils/__init__.py b/cmdstanpy/utils/__init__.py index b4e6c9ab..76301f63 100644 --- a/cmdstanpy/utils/__init__.py +++ b/cmdstanpy/utils/__init__.py @@ -8,12 +8,18 @@ from .cmdstan import ( EXTENSION, + RTOOLS_LAYOUTS, + RToolsLayout, cmdstan_path, cmdstan_version, cmdstan_version_before, cxx_toolchain_path, + determine_windows_arch, get_latest_cmdstan, install_cmdstan, + normalize_rtools_version, + rtools_compiler, + rtools_layouts, set_cmdstan_path, set_make_env, validate_cmdstan_path, @@ -98,6 +104,8 @@ def show_versions(output: bool = True) -> str: __all__ = [ 'EXTENSION', + 'RTOOLS_LAYOUTS', + 'RToolsLayout', 'SanitizedOrTmpFilePath', 'build_xarray_data', 'check_sampler_csv', @@ -106,15 +114,19 @@ def show_versions(output: bool = True) -> str: 'cmdstan_version_before', 'create_named_text_file', 'cxx_toolchain_path', + 'determine_windows_arch', 'do_command', 'flatten_chains', 'get_latest_cmdstan', 'get_logger', 'install_cmdstan', + 'normalize_rtools_version', 'parse_rdump_value', 'pushd', 'read_metric', 'rload', + 'rtools_compiler', + 'rtools_layouts', 'set_cmdstan_path', 'set_make_env', 'show_versions', diff --git a/cmdstanpy/utils/cmdstan.py b/cmdstanpy/utils/cmdstan.py index 77c1894d..6f23875e 100644 --- a/cmdstanpy/utils/cmdstan.py +++ b/cmdstanpy/utils/cmdstan.py @@ -2,12 +2,13 @@ Utilities for finding and installing CmdStan """ +import logging import os import platform import subprocess import sys from collections import OrderedDict -from typing import Callable +from typing import Callable, NamedTuple from tqdm.auto import tqdm @@ -20,6 +21,24 @@ EXTENSION = '.exe' if platform.system() == 'Windows' else '' +def determine_windows_arch() -> str: + """ + Return the architecture of the running Windows process: + ``'x86_64'``, ``'aarch64'`` or ``'i686'``. + + A CPython built for x86-64 running under emulation on an ARM64 machine + reports ``AMD64``, which is what we want: it needs the Intel RTools. + """ + machine = platform.machine().upper() + if machine == 'ARM64': + return 'aarch64' + if machine in ('AMD64', 'X86_64'): + return 'x86_64' + if machine in ('X86', 'I386', 'I686'): + return 'i686' + return 'x86_64' if sys.maxsize > 2**32 else 'i686' + + def determine_linux_arch() -> str: machine = platform.machine() arch = "" @@ -271,6 +290,194 @@ def cmdstan_version_before( return False +class RToolsLayout(NamedTuple): + """Directory layout of one RTools version/architecture combination.""" + + version: str + arch: str + compiler_subdir: tuple[str, ...] + tool_subdir: tuple[str, ...] + compilers: tuple[str, ...] + + +# Ordered newest-first so that a search without an explicit version +# prefers the most recent layout. RTools 4.2 through 4.5 share a layout; +# the ARM builds ship an LLVM toolchain rather than GCC. +RTOOLS_LAYOUTS = ( + RToolsLayout( + '4.5', + 'aarch64', + ('aarch64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + ('clang++', 'g++'), + ), + RToolsLayout( + '4.5', + 'x86_64', + ('x86_64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + ('g++',), + ), + RToolsLayout( + '4.4', + 'aarch64', + ('aarch64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + ('clang++', 'g++'), + ), + RToolsLayout( + '4.4', + 'x86_64', + ('x86_64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + ('g++',), + ), + RToolsLayout( + '4.3', + 'x86_64', + ('x86_64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + ('g++',), + ), + RToolsLayout( + '4.2', + 'x86_64', + ('x86_64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + ('g++',), + ), + RToolsLayout('4.0', 'x86_64', ('mingw64', 'bin'), ('usr', 'bin'), ('g++',)), + RToolsLayout('4.0', 'i686', ('mingw32', 'bin'), ('usr', 'bin'), ('g++',)), + RToolsLayout('3.5', 'x86_64', ('mingw_64', 'bin'), ('bin',), ('g++',)), + RToolsLayout('3.5', 'i686', ('mingw_32', 'bin'), ('bin',), ('g++',)), +) + +_RTOOLS_VERSION_ALIASES = { + '3': '3.5', + '35': '3.5', + '3.5': '3.5', + '4': '4.0', + '40': '4.0', + '4.0': '4.0', + '42': '4.2', + '4.2': '4.2', + '43': '4.3', + '4.3': '4.3', + '44': '4.4', + '4.4': '4.4', + '45': '4.5', + '4.5': '4.5', +} + +_RTOOLS_HOME_VARS = ( + 'RTOOLS45_HOME', + 'RTOOLS44_HOME', + 'RTOOLS43_HOME', + 'RTOOLS42_HOME', + 'RTOOLS40_HOME', +) + +_RTOOLS_DIR_NAMES = ( + 'RTools45', + 'RTools44', + 'RTools43', + 'RTools42', + 'RTools40', + 'RTools35', + 'RTools30', + 'RTools', +) + + +def normalize_rtools_version(version: str) -> str: + """Return the ``maj.min`` form of an RTools version string.""" + return _RTOOLS_VERSION_ALIASES.get(version, version) + + +def rtools_layouts( + version: str | None = None, arch: str | None = None +) -> list[RToolsLayout]: + """ + Return the known RTools directory layouts matching a version and + architecture, newest first. An empty list means the combination is + not supported. + """ + if arch is None: + arch = determine_windows_arch() + return [ + layout + for layout in RTOOLS_LAYOUTS + if layout.arch == arch + and (version is None or layout.version == version) + ] + + +def _rtools_dir_names(arch: str) -> list[str]: + names = [] + for name in _RTOOLS_DIR_NAMES: + if arch == 'aarch64': + # CRAN installs the ARM builds to e.g. C:\rtools45-aarch64 + names.append(f'{name}-aarch64') + names.append(name) + return names + + +def _rtools_search_roots(install_dir: str | None, arch: str) -> list[str]: + """Candidate RTools installation roots, in order of preference.""" + roots = [ + home + for home in (os.environ.get(var) for var in _RTOOLS_HOME_VARS) + if home + ] + names = _rtools_dir_names(arch) + parents = [] + if install_dir is not None: + parents.append(install_dir) + parents.append(os.path.expanduser(os.path.join('~', _DOT_CMDSTAN))) + parents.append(os.path.abspath('/')) + for parent in parents: + roots.extend(os.path.join(parent, name) for name in names) + roots.append(os.path.join(os.path.abspath('/'), 'RBuildTools')) + return roots + + +def _probe_rtools_root( + toolchain_root: str, + layouts: list[RToolsLayout], + logger: logging.Logger, +) -> tuple[str, str, str] | None: + """ + Match an installation root against the given layouts, returning + ``(root, compiler_path, tool_path)`` or ``None``. + """ + if not toolchain_root or not os.path.exists(toolchain_root): + return None + for layout in layouts: + compiler_path = os.path.join(toolchain_root, *layout.compiler_subdir) + if not os.path.exists(compiler_path): + continue + tool_path = os.path.join(toolchain_root, *layout.tool_subdir) + if os.path.exists(tool_path): + return toolchain_root, compiler_path, tool_path + logger.warning( + 'Found invalid RTools installation on %s: missing %s', + toolchain_root, + tool_path, + ) + return None + logger.warning('Found no usable RTools installation on %s', toolchain_root) + return None + + +def rtools_compiler(toolchain_root: str, layout: RToolsLayout) -> str | None: + """Return the name of the C++ compiler shipped in an installation.""" + compiler_dir = os.path.join(toolchain_root, *layout.compiler_subdir) + for compiler in layout.compilers: + if os.path.exists(os.path.join(compiler_dir, compiler + EXTENSION)): + return compiler + return None + + def cxx_toolchain_path( version: str | None = None, install_dir: str | None = None ) -> tuple[str, ...]: @@ -284,143 +491,33 @@ def cxx_toolchain_path( if version is not None and not isinstance(version, str): raise TypeError('Format version number as a string') logger = get_logger() - if 'CMDSTAN_TOOLCHAIN' in os.environ: - toolchain_root = os.environ['CMDSTAN_TOOLCHAIN'] - if os.path.exists(os.path.join(toolchain_root, 'mingw64')): - compiler_path = os.path.join( - toolchain_root, - 'mingw64' if (sys.maxsize > 2**32) else 'mingw32', - 'bin', - ) - if os.path.exists(compiler_path): - tool_path = os.path.join(toolchain_root, 'usr', 'bin') - if not os.path.exists(tool_path): - tool_path = '' - compiler_path = '' - logger.warning( - 'Found invalid installion for RTools40 on %s', - toolchain_root, - ) - toolchain_root = '' - else: - compiler_path = '' - logger.warning( - 'Found invalid installion for RTools40 on %s', - toolchain_root, - ) - toolchain_root = '' - elif os.path.exists(os.path.join(toolchain_root, 'mingw_64')): - compiler_path = os.path.join( - toolchain_root, - 'mingw_64' if (sys.maxsize > 2**32) else 'mingw_32', - 'bin', - ) - if os.path.exists(compiler_path): - tool_path = os.path.join(toolchain_root, 'bin') - if not os.path.exists(tool_path): - tool_path = '' - compiler_path = '' - logger.warning( - 'Found invalid installion for RTools35 on %s', - toolchain_root, - ) - toolchain_root = '' - else: - compiler_path = '' - logger.warning( - 'Found invalid installion for RTools35 on %s', - toolchain_root, - ) - toolchain_root = '' + arch = determine_windows_arch() + layouts = rtools_layouts( + normalize_rtools_version(version) if version else None, arch + ) + if not layouts: + raise ValueError(f'unsupported RTools version: {version}') + + found = None + if 'CMDSTAN_TOOLCHAIN' in os.environ: + found = _probe_rtools_root( + os.environ['CMDSTAN_TOOLCHAIN'], layouts, logger + ) else: - rtools40_home = os.environ.get('RTOOLS40_HOME') - cmdstan_dir = os.path.expanduser(os.path.join('~', _DOT_CMDSTAN)) - for toolchain_root in ( - ([rtools40_home] if rtools40_home is not None else []) - + ( - [ - os.path.join(install_dir, 'RTools40'), - os.path.join(install_dir, 'RTools35'), - os.path.join(install_dir, 'RTools30'), - os.path.join(install_dir, 'RTools'), - ] - if install_dir is not None - else [] - ) - + [ - os.path.join(cmdstan_dir, 'RTools40'), - os.path.join(os.path.abspath("/"), "RTools40"), - os.path.join(cmdstan_dir, 'RTools35'), - os.path.join(os.path.abspath("/"), "RTools35"), - os.path.join(cmdstan_dir, 'RTools'), - os.path.join(os.path.abspath("/"), "RTools"), - os.path.join(os.path.abspath("/"), "RBuildTools"), - ] - ): - compiler_path = '' - tool_path = '' - - if os.path.exists(toolchain_root): - if version not in ('35', '3.5', '3'): - compiler_path = os.path.join( - toolchain_root, - 'mingw64' if (sys.maxsize > 2**32) else 'mingw32', - 'bin', - ) - if os.path.exists(compiler_path): - tool_path = os.path.join(toolchain_root, 'usr', 'bin') - if not os.path.exists(tool_path): - tool_path = '' - compiler_path = '' - logger.warning( - 'Found invalid installation for RTools40 on %s', - toolchain_root, - ) - toolchain_root = '' - else: - break - else: - compiler_path = '' - logger.warning( - 'Found invalid installation for RTools40 on %s', - toolchain_root, - ) - toolchain_root = '' - else: - compiler_path = os.path.join( - toolchain_root, - 'mingw_64' if (sys.maxsize > 2**32) else 'mingw_32', - 'bin', - ) - if os.path.exists(compiler_path): - tool_path = os.path.join(toolchain_root, 'bin') - if not os.path.exists(tool_path): - tool_path = '' - compiler_path = '' - logger.warning( - 'Found invalid installation for RTools35 on %s', - toolchain_root, - ) - toolchain_root = '' - else: - break - else: - compiler_path = '' - logger.warning( - 'Found invalid installation for RTools35 on %s', - toolchain_root, - ) - toolchain_root = '' - else: - toolchain_root = '' - - if not toolchain_root: + for toolchain_root in _rtools_search_roots(install_dir, arch): + found = _probe_rtools_root(toolchain_root, layouts, logger) + if found is not None: + break + + if found is None: raise ValueError( 'no RTools toolchain installation found, ' 'run command line script ' '"python -m cmdstanpy.install_cxx_toolchain"' ) + toolchain_root, compiler_path, tool_path = found + logger.info('Add C++ toolchain to $PATH: %s', toolchain_root) os.environ['PATH'] = ';'.join( list( From bafe58d77f492b543a96d9b51cb82410aa513d62 Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Thu, 17 Sep 2026 15:22:02 +0100 Subject: [PATCH 02/14] Add tests for Windows toolchain installation --- test/test_cxx_installation.py | 427 +++++++++++++++++++++++++++++++++- 1 file changed, 418 insertions(+), 9 deletions(-) diff --git a/test/test_cxx_installation.py b/test/test_cxx_installation.py index bc38c46d..39b1d79b 100644 --- a/test/test_cxx_installation.py +++ b/test/test_cxx_installation.py @@ -1,10 +1,395 @@ """install_cxx_toolchain tests""" +# pylint: disable=redefined-outer-name + +import os +import platform +import sys +from pathlib import Path from test import mark_not_windows, mark_windows_only +from typing import Callable import pytest from cmdstanpy import install_cxx_toolchain +from cmdstanpy.utils import cxx_toolchain_path + +SetArch = Callable[[str], None] + +# (rtools version, arch) -> (compiler subdir, tool subdir, compiler exe) +LAYOUTS = { + ('3.5', 'x86_64'): (('mingw_64', 'bin'), ('bin',), 'g++'), + ('4.0', 'x86_64'): (('mingw64', 'bin'), ('usr', 'bin'), 'g++'), + ('4.4', 'x86_64'): ( + ('x86_64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + 'g++', + ), + ('4.5', 'x86_64'): ( + ('x86_64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + 'g++', + ), + ('4.4', 'aarch64'): ( + ('aarch64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + 'clang++', + ), + ('4.5', 'aarch64'): ( + ('aarch64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + 'clang++', + ), +} + +MACHINES = {'x86_64': 'AMD64', 'aarch64': 'ARM64', 'i686': 'x86'} + + +@pytest.fixture(autouse=True) +def clean_env( + monkeypatch: pytest.MonkeyPatch, tmp_path_factory: pytest.TempPathFactory +) -> None: + """Hide any RTools installation the machine already has.""" + for var in ( + 'CMDSTAN_TOOLCHAIN', + 'RTOOLS45_HOME', + 'RTOOLS44_HOME', + 'RTOOLS43_HOME', + 'RTOOLS42_HOME', + 'RTOOLS40_HOME', + ): + monkeypatch.delenv(var, raising=False) + # cxx_toolchain_path mutates PATH in place; let monkeypatch restore it + monkeypatch.setenv('PATH', os.environ.get('PATH', '')) + home = tmp_path_factory.mktemp('home') + monkeypatch.setenv('HOME', str(home)) + monkeypatch.setenv('USERPROFILE', str(home)) + + +@pytest.fixture +def set_arch(monkeypatch: pytest.MonkeyPatch) -> SetArch: + """Pretend to be running on a given architecture.""" + + def _set(arch: str) -> None: + monkeypatch.setattr(platform, 'machine', lambda: MACHINES[arch]) + + return _set + + +def make_toolchain( + root: Path, version: str, arch: str, complete: bool = True +) -> str: + """Create a fake RTools installation tree, return its root.""" + compiler_subdir, tool_subdir, compiler = LAYOUTS[(version, arch)] + compiler_dir = root.joinpath(*compiler_subdir) + compiler_dir.mkdir(parents=True) + if complete: + compiler_dir.joinpath(compiler + '.exe').write_text('') + root.joinpath(*tool_subdir).mkdir(parents=True, exist_ok=True) + return str(root) + + +# --------------------------------------------------------------------------- +# version / url resolution +# --------------------------------------------------------------------------- + + +@mark_windows_only +@pytest.mark.parametrize( + 'given,expected', + [ + ('4.0', '4.0'), + ('4', '4.0'), + ('40', '4.0'), + ('3.5', '3.5'), + ('35', '3.5'), + ('3', '3.5'), + ('4.2', '4.2'), + ('42', '4.2'), + ('4.4', '4.4'), + ('44', '4.4'), + ('4.5', '4.5'), + ('45', '4.5'), + ], +) +def test_normalize_version(given: str, expected: str) -> None: + assert install_cxx_toolchain.normalize_version(given) == expected + + +@mark_windows_only +def test_toolchain_name() -> None: + assert install_cxx_toolchain.get_toolchain_name() == 'RTools' + + +@mark_windows_only +@pytest.mark.parametrize( + 'version,arch,expected', + [ + ( + '4.5', + 'aarch64', + 'https://cran.r-project.org/bin/windows/Rtools/rtools45/files/' + 'rtools45-aarch64-6768-6492.exe', + ), + ( + '4.5', + 'x86_64', + 'https://cran.r-project.org/bin/windows/Rtools/rtools45/files/' + 'rtools45-6768-6492.exe', + ), + ( + '4.4', + 'aarch64', + 'https://cran.r-project.org/bin/windows/Rtools/rtools44/files/' + 'rtools44-aarch64-6459-6401.exe', + ), + ( + '4.4', + 'x86_64', + 'https://cran.r-project.org/bin/windows/Rtools/rtools44/files/' + 'rtools44-6459-6401.exe', + ), + ( + '4.0', + 'x86_64', + 'https://cran.r-project.org/bin/windows/Rtools/' + 'rtools40-x86_64.exe', + ), + ( + '3.5', + 'x86_64', + 'https://cran.r-project.org/bin/windows/Rtools/Rtools35.exe', + ), + # no ARM64 builds exist before RTools 4.4 + ('4.0', 'aarch64', ''), + ('3.5', 'aarch64', ''), + ], +) +def test_get_url(version: str, arch: str, expected: str) -> None: + assert install_cxx_toolchain.get_url(version, arch) == expected + + +@mark_windows_only +@pytest.mark.parametrize('arch', ['x86_64', 'aarch64']) +def test_get_url_defaults_to_current_arch(set_arch: SetArch, arch: str) -> None: + set_arch(arch) + assert install_cxx_toolchain.get_url( + '4.5' + ) == install_cxx_toolchain.get_url('4.5', arch) + + +@mark_windows_only +def test_get_url_unsupported_arch() -> None: + """RTools 4.4 and later dropped 32-bit builds.""" + assert install_cxx_toolchain.get_url('4.5', 'i686') == '' + + +@mark_windows_only +@pytest.mark.parametrize( + 'arch,expected', [('x86_64', '4.0'), ('aarch64', '4.5')] +) +def test_latest_version(set_arch: SetArch, arch: str, expected: str) -> None: + set_arch(arch) + assert install_cxx_toolchain.latest_version() == expected + + +@mark_windows_only +@pytest.mark.parametrize( + 'arch,expected', [('x86_64', 'RTools44'), ('aarch64', 'RTools44-aarch64')] +) +def test_toolchain_folder(set_arch: SetArch, arch: str, expected: str) -> None: + set_arch(arch) + assert ( + install_cxx_toolchain.get_toolchain_version('RTools', '4.4') == expected + ) + + +# --------------------------------------------------------------------------- +# is_installed +# --------------------------------------------------------------------------- + + +@mark_windows_only +@pytest.mark.parametrize('version,arch', sorted(LAYOUTS)) +def test_is_installed( + set_arch: SetArch, tmp_path: Path, version: str, arch: str +) -> None: + set_arch(arch) + root = make_toolchain(tmp_path, version, arch) + assert install_cxx_toolchain.is_installed(root, version) + + +@mark_windows_only +@pytest.mark.parametrize('version,arch', sorted(LAYOUTS)) +def test_is_installed_incomplete( + set_arch: SetArch, tmp_path: Path, version: str, arch: str +) -> None: + """Compiler directory exists but the compiler itself is missing.""" + set_arch(arch) + root = make_toolchain(tmp_path, version, arch, complete=False) + assert not install_cxx_toolchain.is_installed(root, version) + + +@mark_windows_only +def test_is_installed_missing_compiler( + set_arch: SetArch, tmp_path: Path +) -> None: + """Both directories present, but no compiler binary in them.""" + set_arch('aarch64') + make_toolchain(tmp_path, '4.4', 'aarch64', complete=False) + tmp_path.joinpath('usr', 'bin').mkdir(parents=True) + assert not install_cxx_toolchain.is_installed(str(tmp_path), '4.4') + + +@mark_windows_only +def test_is_installed_wrong_arch(set_arch: SetArch, tmp_path: Path) -> None: + """An ARM64 install must not be reported to an x86_64 process.""" + set_arch('aarch64') + root = make_toolchain(tmp_path, '4.4', 'aarch64') + set_arch('x86_64') + assert not install_cxx_toolchain.is_installed(root, '4.4') + + +@mark_windows_only +def test_is_installed_unknown_version( + set_arch: SetArch, tmp_path: Path +) -> None: + set_arch('x86_64') + root = make_toolchain(tmp_path, '4.4', 'x86_64') + assert not install_cxx_toolchain.is_installed(root, '9.9') + + +# --------------------------------------------------------------------------- +# cxx_toolchain_path +# --------------------------------------------------------------------------- + + +@mark_windows_only +@pytest.mark.parametrize('version,arch', sorted(LAYOUTS)) +def test_toolchain_path_from_install_dir( + set_arch: SetArch, tmp_path: Path, version: str, arch: str +) -> None: + set_arch(arch) + folder = install_cxx_toolchain.get_toolchain_version('RTools', version) + make_toolchain(tmp_path / folder, version, arch) + + compiler_path, tool_path = cxx_toolchain_path(version, str(tmp_path)) + + compiler_subdir, tool_subdir, _ = LAYOUTS[(version, arch)] + assert compiler_path == str(tmp_path.joinpath(folder, *compiler_subdir)) + assert tool_path == str(tmp_path.joinpath(folder, *tool_subdir)) + assert os.environ['PATH'].startswith(f'{compiler_path};{tool_path};') + + +@mark_windows_only +@pytest.mark.parametrize('version,arch', sorted(LAYOUTS)) +def test_toolchain_path_from_env( + monkeypatch: pytest.MonkeyPatch, + set_arch: SetArch, + tmp_path: Path, + version: str, + arch: str, +) -> None: + set_arch(arch) + root = make_toolchain(tmp_path, version, arch) + monkeypatch.setenv('CMDSTAN_TOOLCHAIN', root) + + compiler_subdir, tool_subdir, _ = LAYOUTS[(version, arch)] + assert cxx_toolchain_path() == ( + str(tmp_path.joinpath(*compiler_subdir)), + str(tmp_path.joinpath(*tool_subdir)), + ) + + +@mark_windows_only +@pytest.mark.parametrize('arch', ['x86_64', 'aarch64']) +def test_toolchain_path_unversioned_prefers_newest( + set_arch: SetArch, tmp_path: Path, arch: str +) -> None: + """With no version requested, 4.5 wins over 4.4.""" + set_arch(arch) + old = install_cxx_toolchain.get_toolchain_version('RTools', '4.4') + new = install_cxx_toolchain.get_toolchain_version('RTools', '4.5') + make_toolchain(tmp_path / old, '4.4', arch) + make_toolchain(tmp_path / new, '4.5', arch) + + compiler_path, _ = cxx_toolchain_path(None, str(tmp_path)) + assert new in compiler_path + + +@mark_windows_only +def test_toolchain_path_rtools_home( + monkeypatch: pytest.MonkeyPatch, set_arch: SetArch, tmp_path: Path +) -> None: + set_arch('x86_64') + root = make_toolchain(tmp_path, '4.4', 'x86_64') + monkeypatch.setenv('RTOOLS44_HOME', root) + compiler_path, _ = cxx_toolchain_path() + assert compiler_path.startswith(root) + + +@mark_windows_only +def test_toolchain_path_incomplete_warns( + monkeypatch: pytest.MonkeyPatch, + set_arch: SetArch, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + set_arch('x86_64') + root = make_toolchain(tmp_path, '4.4', 'x86_64', complete=False) + monkeypatch.setenv('CMDSTAN_TOOLCHAIN', root) + with pytest.raises(ValueError, match='no RTools toolchain installation'): + cxx_toolchain_path() + assert 'Found invalid RTools installation' in caplog.text + + +@mark_windows_only +def test_toolchain_path_unrecognized_root( + monkeypatch: pytest.MonkeyPatch, + set_arch: SetArch, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """A directory that exists but matches no known layout.""" + set_arch('x86_64') + monkeypatch.setenv('CMDSTAN_TOOLCHAIN', str(tmp_path)) + with pytest.raises(ValueError, match='no RTools toolchain installation'): + cxx_toolchain_path() + assert 'Found no usable RTools installation' in caplog.text + + +@mark_windows_only +def test_toolchain_path_not_found( + monkeypatch: pytest.MonkeyPatch, set_arch: SetArch, tmp_path: Path +) -> None: + set_arch('x86_64') + monkeypatch.setenv('CMDSTAN_TOOLCHAIN', str(tmp_path / 'nowhere')) + with pytest.raises(ValueError, match='no RTools toolchain installation'): + cxx_toolchain_path() + + +@mark_windows_only +def test_toolchain_path_unsupported_version(set_arch: SetArch) -> None: + set_arch('aarch64') + with pytest.raises(ValueError, match='unsupported RTools version'): + cxx_toolchain_path('4.0') + + +@mark_windows_only +def test_toolchain_path_bad_version_type() -> None: + with pytest.raises(TypeError, match='Format version number as a string'): + cxx_toolchain_path(4.0) # type: ignore[arg-type] + + +@mark_not_windows +def test_cxx_toolchain_path_not_windows() -> None: + with pytest.raises(RuntimeError, match='only supported on Windows'): + cxx_toolchain_path() + + +# --------------------------------------------------------------------------- +# misc +# --------------------------------------------------------------------------- @mark_windows_only @@ -40,17 +425,41 @@ def test_install_not_windows() -> None: @mark_windows_only -def test_normalize_version() -> None: - """Test supported versions.""" +def test_install_unsupported_combination( + set_arch: SetArch, tmp_path: Path +) -> None: + """RTools 4.0 has no ARM64 build.""" + set_arch('aarch64') + with pytest.raises(ValueError, match='not available for aarch64'): + install_cxx_toolchain.run_rtools_install( + {'version': '4.0', 'dir': str(tmp_path)} + ) - for ver in ['4.0', '4', '40']: - assert install_cxx_toolchain.normalize_version(ver) == '4.0' - for ver in ['3.5', '35']: - assert install_cxx_toolchain.normalize_version(ver) == '3.5' +@mark_windows_only +def test_install_defaults_to_latest_version( + monkeypatch: pytest.MonkeyPatch, set_arch: SetArch, tmp_path: Path +) -> None: + set_arch('x86_64') + monkeypatch.setattr(install_cxx_toolchain, 'latest_version', lambda: '9.9') + with pytest.raises(ValueError, match='RTools 9.9 is not available'): + install_cxx_toolchain.run_rtools_install( + {'version': None, 'dir': str(tmp_path)} + ) @mark_windows_only -def test_toolchain_name() -> None: - """Check toolchain name.""" - assert install_cxx_toolchain.get_toolchain_name() == 'RTools' +def test_usage(capsys: pytest.CaptureFixture) -> None: + install_cxx_toolchain.usage() + assert '--version' in capsys.readouterr().out + + +def test_parse_cmdline_args(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + sys, 'argv', ['install_cxx_toolchain', '-v', '4.5', '-d', 'somewhere'] + ) + args = install_cxx_toolchain.parse_cmdline_args() + assert args['version'] == '4.5' + assert args['dir'] == 'somewhere' + assert args['silent'] is False + assert args['progress'] is False From 6717c2d8ed564b29686602827882d69897191c73 Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Thu, 17 Sep 2026 15:23:11 +0100 Subject: [PATCH 03/14] Add workflow for testing Windows toolchain install --- .github/workflows/windows-toolchain.yml | 134 ++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 .github/workflows/windows-toolchain.yml diff --git a/.github/workflows/windows-toolchain.yml b/.github/workflows/windows-toolchain.yml new file mode 100644 index 00000000..ea163dd0 --- /dev/null +++ b/.github/workflows/windows-toolchain.yml @@ -0,0 +1,134 @@ +name: Windows toolchain + +# Exercises the RTools install + discovery path on real Windows runners. +# Kept out of the main matrix because the RTools 4.4/4.5 installers are ~450MB. + +on: + workflow_dispatch: + schedule: + # weekly: catches CRAN retiring the pinned installer revisions + - cron: "0 6 * * 1" + pull_request: + paths: + - "cmdstanpy/install_cxx_toolchain.py" + - "cmdstanpy/utils/cmdstan.py" + - "cmdstanpy/install_cmdstan.py" + - "test/test_cxx_installation.py" + - ".github/workflows/windows-toolchain.yml" + +jobs: + # Fast, hardware-independent: the layout table is exercised with fake trees. + unit: + name: layout unit tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-latest, windows-11-arm] + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - run: python -m pip install --upgrade pip + - run: python -m pip install .[test] + - run: python -m pytest test/test_cxx_installation.py -v + + install: + name: RTools ${{ matrix.rtools }} on ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: windows-latest + arch: x86_64 + rtools: "4.0" + compiler: g++ + - os: windows-latest + arch: x86_64 + rtools: "4.4" + compiler: g++ + - os: windows-latest + arch: x86_64 + rtools: "4.5" + compiler: g++ + - os: windows-11-arm + arch: aarch64 + rtools: "4.4" + compiler: clang++ + - os: windows-11-arm + arch: aarch64 + rtools: "4.5" + compiler: clang++ + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Install package + run: | + python -m pip install --upgrade pip + python -m pip install . + + - name: Report detected architecture + run: | + python -c "import platform; from cmdstanpy.utils import determine_windows_arch as d; print(platform.machine(), d()); assert d() == '${{ matrix.arch }}'" + + # The runner images ship their own RTools; make sure discovery only sees + # what this job installed. + - name: Clear pre-existing RTools environment + run: | + "RTOOLS40_HOME=" >> $env:GITHUB_ENV + "RTOOLS42_HOME=" >> $env:GITHUB_ENV + "RTOOLS43_HOME=" >> $env:GITHUB_ENV + "RTOOLS44_HOME=" >> $env:GITHUB_ENV + "RTOOLS45_HOME=" >> $env:GITHUB_ENV + + - name: Install RTools ${{ matrix.rtools }} + run: | + python -m cmdstanpy.install_cxx_toolchain --version ${{ matrix.rtools }} --dir ${{ runner.temp }}\toolchain --verbose + + - name: Discover and activate toolchain + shell: python + run: | + import os, shutil, subprocess + from cmdstanpy.install_cxx_toolchain import get_toolchain_version, is_installed + from cmdstanpy.utils import cxx_toolchain_path + + install_dir = os.path.join(os.environ["RUNNER_TEMP"], "toolchain") + version = "${{ matrix.rtools }}" + folder = get_toolchain_version("RTools", version) + print("expecting install folder:", folder) + assert is_installed(os.path.join(install_dir, folder), version) + + compiler_path, tool_path = cxx_toolchain_path(version, install_dir) + print("compiler:", compiler_path) + print("tools: ", tool_path) + assert compiler_path.startswith(install_dir), compiler_path + + cxx = shutil.which("${{ matrix.compiler }}") + assert cxx and cxx.startswith(compiler_path), cxx + subprocess.run([cxx, "--version"], check=True) + + make = shutil.which("mingw32-make") or shutil.which("make") + assert make, "no make found on PATH" + subprocess.run([make, "--version"], check=True) + + - name: Compile a trivial C++ program + shell: python + run: | + import os, subprocess, textwrap + from cmdstanpy.utils import cxx_toolchain_path + + cxx_toolchain_path("${{ matrix.rtools }}", os.path.join(os.environ["RUNNER_TEMP"], "toolchain")) + with open("hello.cpp", "w") as f: + f.write(textwrap.dedent(""" + #include + #include + int main() { std::cout << "hello\\n"; return 0; } + """)) + subprocess.run(["${{ matrix.compiler }}", "-O0", "hello.cpp", "-o", "hello.exe"], check=True) + subprocess.run(["./hello.exe"], check=True) From 65db8cb8664958f7a30453a634ae2ac849dc793d Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Thu, 17 Sep 2026 22:20:45 +0100 Subject: [PATCH 04/14] Add Windows ARM64 support --- .github/workflows/main.yml | 12 ++- .github/workflows/windows-toolchain.yml | 17 ++-- cmdstanpy/compilation.py | 17 ++-- cmdstanpy/install_cmdstan.py | 96 +++++++++--------- cmdstanpy/install_cxx_toolchain.py | 23 ++--- cmdstanpy/utils/__init__.py | 2 + cmdstanpy/utils/cmdstan.py | 60 ++++++++--- test/test_compilation.py | 12 +++ test/test_cxx_installation.py | 127 ++++++++++++++++++++---- 9 files changed, 248 insertions(+), 118 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7c14cc7e..8d16fce2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,8 +43,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest, windows-11-arm] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + exclude: + # CPython has no Windows ARM64 release before 3.11 + - os: windows-11-arm + python-version: "3.10" env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} steps: @@ -80,21 +84,21 @@ jobs: uses: actions/cache@v6 with: path: ~/.cmdstan - key: ${{ runner.os }}-cmdstan-${{ needs.get-cmdstan-version.outputs.version }}-${{ hashFiles('**/install_cmdstan.py') }} + key: ${{ runner.os }}-${{ runner.arch }}-cmdstan-${{ needs.get-cmdstan-version.outputs.version }}-${{ hashFiles('**/install_cmdstan.py') }} - name: Delete precompiled header (MacOS) if: matrix.os == 'macos-latest' && steps.cache-cmdstan.outputs.cache-hit == 'true' run: rm -rf ~/.cmdstan/cmdstan-${{ needs.get-cmdstan-version.outputs.version }}/stan/src/stan/model/*.hpp.gch - name: Install CmdStan (Linux, macOS) - if: matrix.os != 'windows-latest' + if: runner.os != 'Windows' run: | install_cmdstan -h install_cxx_toolchain -h python -c "import cmdstanpy; cmdstanpy.install_cmdstan(version='${{ needs.get-cmdstan-version.outputs.version }}', cores=4)" - name: Install CmdStan (Windows) - if: matrix.os == 'windows-latest' + if: runner.os == 'Windows' run: | install_cmdstan -h install_cxx_toolchain -h diff --git a/.github/workflows/windows-toolchain.yml b/.github/workflows/windows-toolchain.yml index ea163dd0..b07912fe 100644 --- a/.github/workflows/windows-toolchain.yml +++ b/.github/workflows/windows-toolchain.yml @@ -44,23 +44,18 @@ jobs: - os: windows-latest arch: x86_64 rtools: "4.0" - compiler: g++ - os: windows-latest arch: x86_64 rtools: "4.4" - compiler: g++ - os: windows-latest arch: x86_64 rtools: "4.5" - compiler: g++ - os: windows-11-arm arch: aarch64 rtools: "4.4" - compiler: clang++ - os: windows-11-arm arch: aarch64 rtools: "4.5" - compiler: clang++ steps: - uses: actions/checkout@v7 @@ -96,7 +91,7 @@ jobs: run: | import os, shutil, subprocess from cmdstanpy.install_cxx_toolchain import get_toolchain_version, is_installed - from cmdstanpy.utils import cxx_toolchain_path + from cmdstanpy.utils import cxx_toolchain_path, make_command install_dir = os.path.join(os.environ["RUNNER_TEMP"], "toolchain") version = "${{ matrix.rtools }}" @@ -109,12 +104,14 @@ jobs: print("tools: ", tool_path) assert compiler_path.startswith(install_dir), compiler_path - cxx = shutil.which("${{ matrix.compiler }}") + # CmdStan invokes g++; on ARM64 it is an alias for clang + cxx = shutil.which("g++") assert cxx and cxx.startswith(compiler_path), cxx subprocess.run([cxx, "--version"], check=True) - make = shutil.which("mingw32-make") or shutil.which("make") - assert make, "no make found on PATH" + make = make_command() + print("make:", make, "->", shutil.which(make)) + assert shutil.which(make), f"{make} not found on PATH" subprocess.run([make, "--version"], check=True) - name: Compile a trivial C++ program @@ -130,5 +127,5 @@ jobs: #include int main() { std::cout << "hello\\n"; return 0; } """)) - subprocess.run(["${{ matrix.compiler }}", "-O0", "hello.cpp", "-o", "hello.exe"], check=True) + subprocess.run(["g++", "-O0", "hello.cpp", "-o", "hello.exe"], check=True) subprocess.run(["./hello.exe"], check=True) diff --git a/cmdstanpy/compilation.py b/cmdstanpy/compilation.py index afc75812..a18ce543 100644 --- a/cmdstanpy/compilation.py +++ b/cmdstanpy/compilation.py @@ -5,7 +5,6 @@ import io import json import os -import platform import shutil import subprocess from datetime import datetime @@ -13,7 +12,12 @@ from typing import Any, Iterable from cmdstanpy.utils import get_logger -from cmdstanpy.utils.cmdstan import EXTENSION, cmdstan_path, stanc_path +from cmdstanpy.utils.cmdstan import ( + EXTENSION, + cmdstan_path, + make_command, + stanc_path, +) from cmdstanpy.utils.command import do_command from cmdstanpy.utils.filesystem import SanitizedOrTmpFilePath @@ -195,8 +199,8 @@ def validate_user_header(self) -> None: ) if "allow-undefined" not in self._stanc_options: self._stanc_options["allow-undefined"] = True - # set full path - self._user_header = os.path.abspath(self._user_header) + # clang treats backslashes in the -include path as escapes + self._user_header = Path(self._user_header).absolute().as_posix() if ' ' in self._user_header: raise ValueError( @@ -372,10 +376,7 @@ def compile_stan_file( exe_target, ) - make = os.getenv( - 'MAKE', - 'make' if platform.system() != 'Windows' else 'mingw32-make', - ) + make = make_command() cmd = [make] cmd.extend(compiler_options.compose(filename_in_msg=src.name)) cmd.append(Path(exe_file).as_posix()) diff --git a/cmdstanpy/install_cmdstan.py b/cmdstanpy/install_cmdstan.py index 638021d3..2336297d 100644 --- a/cmdstanpy/install_cmdstan.py +++ b/cmdstanpy/install_cmdstan.py @@ -39,7 +39,9 @@ from cmdstanpy import _DOT_CMDSTAN from cmdstanpy.utils import ( cmdstan_path, + determine_windows_arch, do_command, + make_command, pushd, validate_dir, wrap_url_progress_hook, @@ -71,9 +73,11 @@ def is_windows() -> bool: return platform.system() == 'Windows' -MAKE = os.getenv('MAKE', 'make' if not is_windows() else 'mingw32-make') EXTENSION = '.exe' if is_windows() else '' +# Windows ARM64 needs the Stan Math support added in stan-dev/math#3051 +MIN_WINDOWS_ARM64_VERSION = (2, 35) + def get_headers() -> dict[str, str]: """Create headers dictionary.""" @@ -203,7 +207,7 @@ def overwrite(self) -> bool: def compiler(self) -> bool: if not is_windows(): return False - print("Would you like to install the RTools40 C++ toolchain?") + print("Would you like to install the RTools C++ toolchain?") print("A C++ toolchain is required for CmdStan.") print( "If you are not sure if you need the toolchain or not, " @@ -234,7 +238,7 @@ def clean_all(verbose: bool = False) -> None: :param verbose: Boolean value; when ``True``, show output from make command. """ - cmd = [MAKE, 'clean-all'] + cmd = [make_command(), 'clean-all'] try: if verbose: do_command(cmd) @@ -262,7 +266,7 @@ def build(verbose: bool = False, progress: bool = True, cores: int = 1) -> None: :param cores: Integer, number of cores to use in the ``make`` command. Default is 1 core. """ - cmd = [MAKE, 'build', f'-j{cores}'] + cmd = [make_command(), 'build', f'-j{cores}'] try: if verbose: do_command(cmd) @@ -343,7 +347,7 @@ def compile_example(verbose: bool = False) -> None: if path.is_file(): path.unlink() - cmd = [MAKE, path.as_posix()] + cmd = [make_command(), path.as_posix()] try: if verbose: do_command(cmd) @@ -544,61 +548,50 @@ def retrieve_version(version: str, progress: bool = True) -> None: def run_compiler_install(dir: str, verbose: bool, progress: bool) -> None: - from .install_cxx_toolchain import is_installed as _is_installed_cxx from .install_cxx_toolchain import latest_version as _latest_version_cxx from .install_cxx_toolchain import run_rtools_install as _main_cxx - from .utils import cxx_toolchain_path, determine_windows_arch + from .utils import cxx_toolchain_path - arch = determine_windows_arch() - known_versions = ['4.5', '4.4', '4.3', '4.2', '4.0', '3.5'] + try: + cxx_toolchain_path(None, dir) + return + except ValueError: + pass - compiler_found = False cxx_version = _latest_version_cxx() - homes = [ - home - for home in ( - os.environ.get(var) - for var in ( - 'RTOOLS45_HOME', - 'RTOOLS44_HOME', - 'RTOOLS43_HOME', - 'RTOOLS42_HOME', - 'RTOOLS40_HOME', - ) - ) - if home - ] - names = ['RTools45', 'RTools44', 'RTools40', 'RTools35', 'RTools'] - if arch == 'aarch64': - names = ['RTools45-aarch64', 'RTools44-aarch64'] + names - for cxx_loc in ( - homes - + [home_cmdstan()] - + [os.path.join(os.path.abspath("/"), name) for name in names] - + [os.path.join(os.path.abspath("/"), "RBuildTools")] - ): - for version in known_versions: - if _is_installed_cxx(cxx_loc, version): - cxx_version = version - compiler_found = True - break - if compiler_found: - break - if not compiler_found: - print(f'Installing RTools {cxx_version}') - # copy argv and clear sys.argv - _main_cxx( - { - 'dir': dir, - 'progress': progress, - 'version': None, - 'verbose': verbose, - } - ) + print(f'Installing RTools {cxx_version}') + _main_cxx( + { + 'dir': dir, + 'progress': progress, + 'version': None, + 'verbose': verbose, + } + ) # Add toolchain to $PATH cxx_toolchain_path(cxx_version, dir) +def validate_arm64_support(version: str) -> None: + """Raise if the requested CmdStan predates Windows ARM64 support.""" + if not is_windows() or determine_windows_arch() != 'aarch64': + return + if version.startswith('git:'): + return + try: + parsed = tuple( + int(part) for part in version.split('-')[0].split('.')[:2] + ) + except ValueError: + return + if parsed < MIN_WINDOWS_ARM64_VERSION: + minimum = '.'.join(str(part) for part in MIN_WINDOWS_ARM64_VERSION) + raise ValueError( + f'CmdStan {version} does not support Windows ARM64, ' + f'version {minimum} or later is required.' + ) + + def run_install(args: InteractiveSettings | InstallationSettings) -> None: """ Run a (potentially interactive) installation @@ -636,6 +629,7 @@ def run_install(args: InteractiveSettings | InstallationSettings) -> None: 'Connection to GitHub failed. ' 'Check firewall settings or ensure this version exists.' ) + validate_arm64_support(args.version) shutil.rmtree(cmdstan_version, ignore_errors=True) retrieve_version(args.version, args.progress) install_version( diff --git a/cmdstanpy/install_cxx_toolchain.py b/cmdstanpy/install_cxx_toolchain.py index 68b2c417..743824a7 100644 --- a/cmdstanpy/install_cxx_toolchain.py +++ b/cmdstanpy/install_cxx_toolchain.py @@ -37,17 +37,17 @@ EXTENSION = '.exe' if platform.system() == 'Windows' else '' -# RTools 4.2 and later embed toolchain and installer build revisions in the -# installer filename, so there is no stable URL. Refresh when CRAN publishes -# a new build; see https://cran.r-project.org/bin/windows/Rtools/ +# CRAN embeds build revisions in the RTools 4.2+ installer filenames, so we +# use the r-hub mirror, which publishes them under a stable 'latest' tag. +# These are the builds the CmdStan guide points users at. RTOOLS_INSTALLERS = { '4.5': { - 'x86_64': 'rtools45-6768-6492.exe', - 'aarch64': 'rtools45-aarch64-6768-6492.exe', + 'x86_64': 'rtools45.exe', + 'aarch64': 'rtools45-aarch64.exe', }, '4.4': { - 'x86_64': 'rtools44-6459-6401.exe', - 'aarch64': 'rtools44-aarch64-6459-6401.exe', + 'x86_64': 'rtools44.exe', + 'aarch64': 'rtools44-aarch64.exe', }, } @@ -203,10 +203,7 @@ def latest_version() -> str: """Latest RTools version supported on this machine.""" if platform.system() != 'Windows': return '' - if determine_windows_arch() == 'aarch64': - # RTools 4.0 has no ARM64 build - return '4.5' - return '4.0' + return '4.5' def retrieve_toolchain(filename: str, url: str, progress: bool = True) -> None: @@ -260,8 +257,8 @@ def get_url(version: str, arch: str | None = None) -> str: return '' series = 'rtools' + version.replace('.', '') return ( - 'https://cran.r-project.org/bin/windows/Rtools/' - f'{series}/files/{installer}' + f'https://github.com/r-hub/{series}/releases/' + f'download/latest/{installer}' ) legacy = { ('4.0', 'x86_64'): 'rtools40-x86_64.exe', diff --git a/cmdstanpy/utils/__init__.py b/cmdstanpy/utils/__init__.py index 76301f63..e821a20f 100644 --- a/cmdstanpy/utils/__init__.py +++ b/cmdstanpy/utils/__init__.py @@ -17,6 +17,7 @@ determine_windows_arch, get_latest_cmdstan, install_cmdstan, + make_command, normalize_rtools_version, rtools_compiler, rtools_layouts, @@ -120,6 +121,7 @@ def show_versions(output: bool = True) -> str: 'get_latest_cmdstan', 'get_logger', 'install_cmdstan', + 'make_command', 'normalize_rtools_version', 'parse_rdump_value', 'pushd', diff --git a/cmdstanpy/utils/cmdstan.py b/cmdstanpy/utils/cmdstan.py index 6f23875e..d50dd51c 100644 --- a/cmdstanpy/utils/cmdstan.py +++ b/cmdstanpy/utils/cmdstan.py @@ -5,6 +5,7 @@ import logging import os import platform +import shutil import subprocess import sys from collections import OrderedDict @@ -301,15 +302,15 @@ class RToolsLayout(NamedTuple): # Ordered newest-first so that a search without an explicit version -# prefers the most recent layout. RTools 4.2 through 4.5 share a layout; -# the ARM builds ship an LLVM toolchain rather than GCC. +# prefers the most recent layout. RTools 4.2 through 4.5 share a layout. +# The ARM builds are LLVM-based but ship gcc/g++ aliases for clang. RTOOLS_LAYOUTS = ( RToolsLayout( '4.5', 'aarch64', ('aarch64-w64-mingw32.static.posix', 'bin'), ('usr', 'bin'), - ('clang++', 'g++'), + ('g++', 'clang++'), ), RToolsLayout( '4.5', @@ -323,7 +324,7 @@ class RToolsLayout(NamedTuple): 'aarch64', ('aarch64-w64-mingw32.static.posix', 'bin'), ('usr', 'bin'), - ('clang++', 'g++'), + ('g++', 'clang++'), ), RToolsLayout( '4.4', @@ -441,6 +442,15 @@ def _rtools_search_roots(install_dir: str | None, arch: str) -> list[str]: return roots +def rtools_compiler(toolchain_root: str, layout: RToolsLayout) -> str | None: + """Return the name of the C++ compiler shipped in an installation.""" + compiler_dir = os.path.join(toolchain_root, *layout.compiler_subdir) + for compiler in layout.compilers: + if os.path.exists(os.path.join(compiler_dir, compiler + EXTENSION)): + return compiler + return None + + def _probe_rtools_root( toolchain_root: str, layouts: list[RToolsLayout], @@ -453,9 +463,11 @@ def _probe_rtools_root( if not toolchain_root or not os.path.exists(toolchain_root): return None for layout in layouts: - compiler_path = os.path.join(toolchain_root, *layout.compiler_subdir) - if not os.path.exists(compiler_path): + # RTools 4.2+ ship empty mingw64/ucrt64/clangarm64 stub directories, + # so the compiler binary itself must be checked, not just the dir + if rtools_compiler(toolchain_root, layout) is None: continue + compiler_path = os.path.join(toolchain_root, *layout.compiler_subdir) tool_path = os.path.join(toolchain_root, *layout.tool_subdir) if os.path.exists(tool_path): return toolchain_root, compiler_path, tool_path @@ -469,13 +481,35 @@ def _probe_rtools_root( return None -def rtools_compiler(toolchain_root: str, layout: RToolsLayout) -> str | None: - """Return the name of the C++ compiler shipped in an installation.""" - compiler_dir = os.path.join(toolchain_root, *layout.compiler_subdir) - for compiler in layout.compilers: - if os.path.exists(os.path.join(compiler_dir, compiler + EXTENSION)): - return compiler - return None +def make_command() -> str: + """ + Name of the GNU Make executable to use. + + RTools 4.0 ships ``mingw32-make``, while RTools 4.2 and later ship + plain ``make`` in ``usr/bin``. If neither is on the ``$PATH``, an RTools + installation managed by CmdStanPy is activated before giving up. + """ + make = os.environ.get('MAKE') + if make: + return make + if platform.system() != 'Windows': + return 'make' + + def _found() -> str | None: + for candidate in ('mingw32-make', 'make'): + if shutil.which(candidate): + return candidate + return None + + found = _found() + if found is None: + try: + cxx_toolchain_path() + except ValueError: + pass + else: + found = _found() + return found or 'make' def cxx_toolchain_path( diff --git a/test/test_compilation.py b/test/test_compilation.py index addaea55..4ef5b8c8 100644 --- a/test/test_compilation.py +++ b/test/test_compilation.py @@ -201,6 +201,18 @@ def test_user_header() -> None: opts.validate() +def test_user_header_is_posix_path() -> None: + """Backslashes in the -include path are eaten by clang.""" + opts = CompilerOptions( + user_header=os.path.join(DATAFILES_PATH, 'return_one.hpp') + ) + opts.validate() + assert '\\' not in opts.user_header + assert opts.user_header.endswith('/return_one.hpp') + assert os.path.isfile(opts.user_header) + assert str(opts.cpp_options['USER_HEADER']) == opts.user_header + + def test_model_format_options() -> None: stan = os.path.join(DATAFILES_PATH, 'format_me.stan') diff --git a/test/test_cxx_installation.py b/test/test_cxx_installation.py index 39b1d79b..d508f58e 100644 --- a/test/test_cxx_installation.py +++ b/test/test_cxx_installation.py @@ -12,7 +12,9 @@ import pytest from cmdstanpy import install_cxx_toolchain -from cmdstanpy.utils import cxx_toolchain_path +from cmdstanpy.install_cmdstan import validate_arm64_support +from cmdstanpy.utils import cmdstan as cmdstan_utils +from cmdstanpy.utils import cxx_toolchain_path, make_command SetArch = Callable[[str], None] @@ -77,14 +79,19 @@ def _set(arch: str) -> None: def make_toolchain( - root: Path, version: str, arch: str, complete: bool = True + root: Path, + version: str, + arch: str, + compiler: bool = True, + tools: bool = True, ) -> str: """Create a fake RTools installation tree, return its root.""" - compiler_subdir, tool_subdir, compiler = LAYOUTS[(version, arch)] + compiler_subdir, tool_subdir, compiler_exe = LAYOUTS[(version, arch)] compiler_dir = root.joinpath(*compiler_subdir) - compiler_dir.mkdir(parents=True) - if complete: - compiler_dir.joinpath(compiler + '.exe').write_text('') + compiler_dir.mkdir(parents=True, exist_ok=True) + if compiler: + compiler_dir.joinpath(compiler_exe + '.exe').write_text('') + if tools: root.joinpath(*tool_subdir).mkdir(parents=True, exist_ok=True) return str(root) @@ -128,26 +135,26 @@ def test_toolchain_name() -> None: ( '4.5', 'aarch64', - 'https://cran.r-project.org/bin/windows/Rtools/rtools45/files/' - 'rtools45-aarch64-6768-6492.exe', + 'https://github.com/r-hub/rtools45/releases/download/latest/' + 'rtools45-aarch64.exe', ), ( '4.5', 'x86_64', - 'https://cran.r-project.org/bin/windows/Rtools/rtools45/files/' - 'rtools45-6768-6492.exe', + 'https://github.com/r-hub/rtools45/releases/download/latest/' + 'rtools45.exe', ), ( '4.4', 'aarch64', - 'https://cran.r-project.org/bin/windows/Rtools/rtools44/files/' - 'rtools44-aarch64-6459-6401.exe', + 'https://github.com/r-hub/rtools44/releases/download/latest/' + 'rtools44-aarch64.exe', ), ( '4.4', 'x86_64', - 'https://cran.r-project.org/bin/windows/Rtools/rtools44/files/' - 'rtools44-6459-6401.exe', + 'https://github.com/r-hub/rtools44/releases/download/latest/' + 'rtools44.exe', ), ( '4.0', @@ -186,7 +193,7 @@ def test_get_url_unsupported_arch() -> None: @mark_windows_only @pytest.mark.parametrize( - 'arch,expected', [('x86_64', '4.0'), ('aarch64', '4.5')] + 'arch,expected', [('x86_64', '4.5'), ('aarch64', '4.5')] ) def test_latest_version(set_arch: SetArch, arch: str, expected: str) -> None: set_arch(arch) @@ -226,7 +233,7 @@ def test_is_installed_incomplete( ) -> None: """Compiler directory exists but the compiler itself is missing.""" set_arch(arch) - root = make_toolchain(tmp_path, version, arch, complete=False) + root = make_toolchain(tmp_path, version, arch, compiler=False) assert not install_cxx_toolchain.is_installed(root, version) @@ -236,8 +243,7 @@ def test_is_installed_missing_compiler( ) -> None: """Both directories present, but no compiler binary in them.""" set_arch('aarch64') - make_toolchain(tmp_path, '4.4', 'aarch64', complete=False) - tmp_path.joinpath('usr', 'bin').mkdir(parents=True) + make_toolchain(tmp_path, '4.4', 'aarch64', compiler=False) assert not install_cxx_toolchain.is_installed(str(tmp_path), '4.4') @@ -336,13 +342,34 @@ def test_toolchain_path_incomplete_warns( caplog: pytest.LogCaptureFixture, ) -> None: set_arch('x86_64') - root = make_toolchain(tmp_path, '4.4', 'x86_64', complete=False) + root = make_toolchain(tmp_path, '4.4', 'x86_64', tools=False) monkeypatch.setenv('CMDSTAN_TOOLCHAIN', root) with pytest.raises(ValueError, match='no RTools toolchain installation'): cxx_toolchain_path() assert 'Found invalid RTools installation' in caplog.text +@mark_windows_only +def test_toolchain_path_ignores_empty_stub_dirs( + monkeypatch: pytest.MonkeyPatch, set_arch: SetArch, tmp_path: Path +) -> None: + """RTools 4.2+ ship empty mingw64/ucrt64/clangarm64 directories.""" + set_arch('x86_64') + make_toolchain(tmp_path, '4.5', 'x86_64') + for stub in ('mingw64', 'mingw32', 'ucrt64', 'clang64', 'clangarm64'): + tmp_path.joinpath(stub, 'bin').mkdir(parents=True) + monkeypatch.setenv('CMDSTAN_TOOLCHAIN', str(tmp_path)) + + # an explicit 4.0 request must not match the empty mingw64 stub + with pytest.raises(ValueError, match='no RTools toolchain installation'): + cxx_toolchain_path('4.0') + + compiler_path, _ = cxx_toolchain_path() + assert compiler_path.endswith( + os.path.join('x86_64-w64-mingw32.static.posix', 'bin') + ) + + @mark_windows_only def test_toolchain_path_unrecognized_root( monkeypatch: pytest.MonkeyPatch, @@ -448,6 +475,68 @@ def test_install_defaults_to_latest_version( ) +@mark_windows_only +@pytest.mark.parametrize( + 'arch,version,ok', + [ + ('aarch64', '2.34.1', False), + ('aarch64', '2.35.0', True), + ('aarch64', '2.36.0', True), + ('aarch64', 'git:develop', True), + # the floor only applies to Windows ARM64 + ('x86_64', '2.30.0', True), + ], +) +def test_validate_arm64_support( + set_arch: SetArch, arch: str, version: str, ok: bool +) -> None: + set_arch(arch) + if ok: + validate_arm64_support(version) + else: + with pytest.raises(ValueError, match='does not support Windows ARM64'): + validate_arm64_support(version) + + +# --------------------------------------------------------------------------- +# make resolution +# --------------------------------------------------------------------------- + + +def test_make_command_honours_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv('MAKE', 'my-make') + assert make_command() == 'my-make' + + +@mark_not_windows +def test_make_command_posix(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv('MAKE', raising=False) + assert make_command() == 'make' + + +@mark_windows_only +@pytest.mark.parametrize( + 'available,expected', + [ + # RTools 4.0 installs mingw32-make, which stays preferred + (['mingw32-make', 'make'], 'mingw32-make'), + # RTools 4.2+ only ship plain make in usr/bin + (['make'], 'make'), + ([], 'make'), + ], +) +def test_make_command_windows( + monkeypatch: pytest.MonkeyPatch, available: list[str], expected: str +) -> None: + monkeypatch.delenv('MAKE', raising=False) + monkeypatch.setattr( + cmdstan_utils.shutil, + 'which', + lambda name: f'C:\\fake\\{name}.exe' if name in available else None, + ) + assert make_command() == expected + + @mark_windows_only def test_usage(capsys: pytest.CaptureFixture) -> None: install_cxx_toolchain.usage() From 943d6cd2e65fd58a429e13c6a3fbf88e5a3a2fd2 Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Fri, 18 Sep 2026 09:54:54 +0100 Subject: [PATCH 05/14] Retry executable delete on Windows --- cmdstanpy/compilation.py | 4 ++-- cmdstanpy/utils/__init__.py | 2 ++ cmdstanpy/utils/filesystem.py | 22 ++++++++++++++++++++++ test/test_model.py | 10 +++++----- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/cmdstanpy/compilation.py b/cmdstanpy/compilation.py index a18ce543..ecf7efd9 100644 --- a/cmdstanpy/compilation.py +++ b/cmdstanpy/compilation.py @@ -19,7 +19,7 @@ stanc_path, ) from cmdstanpy.utils.command import do_command -from cmdstanpy.utils.filesystem import SanitizedOrTmpFilePath +from cmdstanpy.utils.filesystem import SanitizedOrTmpFilePath, delete_file STANC_OPTS = [ 'O', @@ -368,7 +368,7 @@ def compile_stan_file( os.remove(hpp_file) if os.path.exists(exe_file): get_logger().debug('Removing %s', exe_file) - os.remove(exe_file) + delete_file(exe_file) get_logger().info( 'compiling stan file %s to exe file %s', diff --git a/cmdstanpy/utils/__init__.py b/cmdstanpy/utils/__init__.py index e821a20f..a691e325 100644 --- a/cmdstanpy/utils/__init__.py +++ b/cmdstanpy/utils/__init__.py @@ -32,6 +32,7 @@ from .filesystem import ( SanitizedOrTmpFilePath, create_named_text_file, + delete_file, pushd, windows_short_path, ) @@ -115,6 +116,7 @@ def show_versions(output: bool = True) -> str: 'cmdstan_version_before', 'create_named_text_file', 'cxx_toolchain_path', + 'delete_file', 'determine_windows_arch', 'do_command', 'flatten_chains', diff --git a/cmdstanpy/utils/filesystem.py b/cmdstanpy/utils/filesystem.py index 6f62d53e..c5b4912a 100644 --- a/cmdstanpy/utils/filesystem.py +++ b/cmdstanpy/utils/filesystem.py @@ -8,6 +8,7 @@ import re import shutil import tempfile +import time from typing import Any, Iterator, Mapping, Sequence import numpy as np @@ -20,6 +21,27 @@ EXTENSION = '.exe' if platform.system() == 'Windows' else '' +def delete_file(path: str, timeout: float = 5.0) -> None: + """ + Delete a file, retrying briefly on Windows. + + Antivirus software scans a binary the first time it is executed and + holds it open while doing so, which makes Windows deny the delete with + ``PermissionError`` until the scan finishes. + """ + deadline = time.monotonic() + timeout + while True: + try: + os.remove(path) + return + except FileNotFoundError: + return + except PermissionError: + if platform.system() != 'Windows' or time.monotonic() > deadline: + raise + time.sleep(0.1) + + def windows_short_path(path: str) -> str: """ Gets the short path name of a given long path. diff --git a/test/test_model.py b/test/test_model.py index 8334560a..6138a2ca 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -12,7 +12,7 @@ import pytest from cmdstanpy.model import CmdStanModel -from cmdstanpy.utils import EXTENSION, cmdstan_version_before +from cmdstanpy.utils import EXTENSION, cmdstan_version_before, delete_file HERE = os.path.dirname(os.path.abspath(__file__)) DATAFILES_PATH = os.path.join(HERE, 'data') @@ -154,7 +154,7 @@ def test_model_info() -> None: info_dict = model.exe_info() assert info_dict['STAN_THREADS'].lower() == 'false' - os.remove(model.exe_file) + delete_file(model.exe_file) with pytest.raises(RuntimeError): model.exe_info() @@ -299,7 +299,7 @@ def test_model_paths() -> None: assert model1.stan_file == dotdot_stan assert model1.exe_file == dotdot_exe os.remove(dotdot_stan) - os.remove(dotdot_exe) + delete_file(dotdot_exe) tilde_stan = os.path.realpath( os.path.join(os.path.expanduser('~'), 'bernoulli.stan') @@ -316,7 +316,7 @@ def test_model_paths() -> None: assert model2.stan_file == tilde_stan assert model2.exe_file == tilde_exe os.remove(tilde_stan) - os.remove(tilde_exe) + delete_file(tilde_exe) def test_model_none() -> None: @@ -362,7 +362,7 @@ def test_model_compile() -> None: @pytest.mark.parametrize("path", ["space in path", "tilde~in~path"]) def test_model_compile_special_char(path: str) -> None: with tempfile.TemporaryDirectory( - prefix="cmdstanpy_testfolder_" + prefix="cmdstanpy_testfolder_", ignore_cleanup_errors=True ) as tmp_path: path_with_special_char = os.path.join(tmp_path, path) os.makedirs(path_with_special_char, exist_ok=True) From d0860d551bcc3f27484299095607e86945b035e5 Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Fri, 18 Sep 2026 10:27:37 +0100 Subject: [PATCH 06/14] Update install docs for Windows --- docsrc/installation.rst | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docsrc/installation.rst b/docsrc/installation.rst index 0f528712..c63c909c 100644 --- a/docsrc/installation.rst +++ b/docsrc/installation.rst @@ -154,10 +154,13 @@ There is usually a pre-installed C++ compiler as well, but not necessarily new e **MacOS** The Xcode and Xcode command line tools must be installed. Xcode is available for free from the Mac App Store. To install the Xcode command line tools, run the shell command: ``xcode-select --install``. -**Windows** We recommend using the `RTools 4.0 `_ toolchain -which contains a ``g++ 8`` compiler and ``Mingw``, the native Windows equivalent of the GNU-Make utility. +**Windows** We recommend using the `RTools 4.5 `_ toolchain, +which contains a C++ compiler and the GNU-Make build utility. This can be installed along with CmdStan when you invoke the function :meth:`cmdstanpy.install_cmdstan` with argument ``compiler=True``. +RTools is available for both Intel/AMD 64-bit (``x86_64``) and ARM 64-bit (``aarch64``) machines; +CmdStanPy selects the matching build automatically. +Older toolchains (RTools 4.0 and 3.5, Intel/AMD only) are still detected if already installed. .. _install-cmdstan-fun: @@ -218,6 +221,17 @@ machine when running ``install_cmdstan``. If the wrong choice is made, or if you need to manually override this, you can set the ``CMDSTAN_ARCH`` environment variable to one of the above options, or to "false" to use the standard x86 download. +Windows on ARM64 +................ + +CmdStan can be built on ARM 64-bit (``aarch64``) Windows machines using the +ARM64 build of RTools 4.4 or 4.5, which CmdStanPy installs and detects +automatically. CmdStan 2.35 or later is required. + +A Python interpreter built for Intel/AMD and running under emulation reports an +``x86_64`` architecture; in that case the Intel/AMD toolchain is used. The two +architectures can be installed side by side. + DIY Installation ^^^^^^^^^^^^^^^^ From 0fe22099f456446a691716639f0c6349aeef8f9d Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Mon, 21 Sep 2026 13:14:39 +0100 Subject: [PATCH 07/14] Remove weekly windows-toolchain test run --- .github/workflows/windows-toolchain.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/windows-toolchain.yml b/.github/workflows/windows-toolchain.yml index b07912fe..b5c67c52 100644 --- a/.github/workflows/windows-toolchain.yml +++ b/.github/workflows/windows-toolchain.yml @@ -5,9 +5,6 @@ name: Windows toolchain on: workflow_dispatch: - schedule: - # weekly: catches CRAN retiring the pinned installer revisions - - cron: "0 6 * * 1" pull_request: paths: - "cmdstanpy/install_cxx_toolchain.py" From 5c5ff30e4fc54c1a75d053e135120a7222fcd5a9 Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Mon, 21 Sep 2026 13:15:23 +0100 Subject: [PATCH 08/14] Move delete_file utility to test --- cmdstanpy/compilation.py | 4 ++-- cmdstanpy/utils/__init__.py | 2 -- cmdstanpy/utils/filesystem.py | 22 ---------------------- test/__init__.py | 24 ++++++++++++++++++++++++ test/test_model.py | 4 ++-- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/cmdstanpy/compilation.py b/cmdstanpy/compilation.py index ecf7efd9..a18ce543 100644 --- a/cmdstanpy/compilation.py +++ b/cmdstanpy/compilation.py @@ -19,7 +19,7 @@ stanc_path, ) from cmdstanpy.utils.command import do_command -from cmdstanpy.utils.filesystem import SanitizedOrTmpFilePath, delete_file +from cmdstanpy.utils.filesystem import SanitizedOrTmpFilePath STANC_OPTS = [ 'O', @@ -368,7 +368,7 @@ def compile_stan_file( os.remove(hpp_file) if os.path.exists(exe_file): get_logger().debug('Removing %s', exe_file) - delete_file(exe_file) + os.remove(exe_file) get_logger().info( 'compiling stan file %s to exe file %s', diff --git a/cmdstanpy/utils/__init__.py b/cmdstanpy/utils/__init__.py index a691e325..e821a20f 100644 --- a/cmdstanpy/utils/__init__.py +++ b/cmdstanpy/utils/__init__.py @@ -32,7 +32,6 @@ from .filesystem import ( SanitizedOrTmpFilePath, create_named_text_file, - delete_file, pushd, windows_short_path, ) @@ -116,7 +115,6 @@ def show_versions(output: bool = True) -> str: 'cmdstan_version_before', 'create_named_text_file', 'cxx_toolchain_path', - 'delete_file', 'determine_windows_arch', 'do_command', 'flatten_chains', diff --git a/cmdstanpy/utils/filesystem.py b/cmdstanpy/utils/filesystem.py index c5b4912a..6f62d53e 100644 --- a/cmdstanpy/utils/filesystem.py +++ b/cmdstanpy/utils/filesystem.py @@ -8,7 +8,6 @@ import re import shutil import tempfile -import time from typing import Any, Iterator, Mapping, Sequence import numpy as np @@ -21,27 +20,6 @@ EXTENSION = '.exe' if platform.system() == 'Windows' else '' -def delete_file(path: str, timeout: float = 5.0) -> None: - """ - Delete a file, retrying briefly on Windows. - - Antivirus software scans a binary the first time it is executed and - holds it open while doing so, which makes Windows deny the delete with - ``PermissionError`` until the scan finishes. - """ - deadline = time.monotonic() + timeout - while True: - try: - os.remove(path) - return - except FileNotFoundError: - return - except PermissionError: - if platform.system() != 'Windows' or time.monotonic() > deadline: - raise - time.sleep(0.1) - - def windows_short_path(path: str) -> str: """ Gets the short path name of a given long path. diff --git a/test/__init__.py b/test/__init__.py index 69af537c..6ced218f 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -2,8 +2,10 @@ import contextlib import logging +import os import platform import re +import time from importlib import reload from types import ModuleType from typing import Generator, Optional, Type @@ -19,6 +21,28 @@ ) +def delete_file(path: str, timeout: float = 5.0) -> None: + """ + Delete a file, retrying briefly on Windows. + + Antivirus software scans a binary the first time it is executed and + holds it open while doing so, which makes Windows deny the delete with + ``PermissionError`` until the scan finishes. Defender is disabled on the + x86_64 CI images but cannot be disabled on the ARM64 ones. + """ + deadline = time.monotonic() + timeout + while True: + try: + os.remove(path) + return + except FileNotFoundError: + return + except PermissionError: + if platform.system() != 'Windows' or time.monotonic() > deadline: + raise + time.sleep(0.1) + + # pylint: disable=invalid-name @contextlib.contextmanager def raises_nested( diff --git a/test/test_model.py b/test/test_model.py index 6138a2ca..b9f4a3ce 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -5,14 +5,14 @@ import re import shutil import tempfile -from test import check_present +from test import check_present, delete_file from unittest.mock import patch import numpy as np import pytest from cmdstanpy.model import CmdStanModel -from cmdstanpy.utils import EXTENSION, cmdstan_version_before, delete_file +from cmdstanpy.utils import EXTENSION, cmdstan_version_before HERE = os.path.dirname(os.path.abspath(__file__)) DATAFILES_PATH = os.path.join(HERE, 'data') From 80c05bdedf6eb182bfdc5b962771c6f03e821819 Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Mon, 21 Sep 2026 13:23:54 +0100 Subject: [PATCH 09/14] Keep user header path normalization for older CmdStan --- cmdstanpy/compilation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmdstanpy/compilation.py b/cmdstanpy/compilation.py index a18ce543..21484c9a 100644 --- a/cmdstanpy/compilation.py +++ b/cmdstanpy/compilation.py @@ -199,7 +199,9 @@ def validate_user_header(self) -> None: ) if "allow-undefined" not in self._stanc_options: self._stanc_options["allow-undefined"] = True - # clang treats backslashes in the -include path as escapes + # CmdStan's make/program does not apply its usual + # $(subst \,/,...) to USER_HEADER, and clang reads the remaining + # backslashes in -include as escapes self._user_header = Path(self._user_header).absolute().as_posix() if ' ' in self._user_header: From d2bb0a9c9fda6ba5c491fdc06c149d66b5bbde51 Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Mon, 21 Sep 2026 13:29:00 +0100 Subject: [PATCH 10/14] Remove Win ARM64-specific CmdStan version check --- cmdstanpy/install_cmdstan.py | 25 ------------------------- docsrc/installation.rst | 2 +- test/test_cxx_installation.py | 24 ------------------------ 3 files changed, 1 insertion(+), 50 deletions(-) diff --git a/cmdstanpy/install_cmdstan.py b/cmdstanpy/install_cmdstan.py index 2336297d..f28f8c13 100644 --- a/cmdstanpy/install_cmdstan.py +++ b/cmdstanpy/install_cmdstan.py @@ -39,7 +39,6 @@ from cmdstanpy import _DOT_CMDSTAN from cmdstanpy.utils import ( cmdstan_path, - determine_windows_arch, do_command, make_command, pushd, @@ -75,9 +74,6 @@ def is_windows() -> bool: EXTENSION = '.exe' if is_windows() else '' -# Windows ARM64 needs the Stan Math support added in stan-dev/math#3051 -MIN_WINDOWS_ARM64_VERSION = (2, 35) - def get_headers() -> dict[str, str]: """Create headers dictionary.""" @@ -572,26 +568,6 @@ def run_compiler_install(dir: str, verbose: bool, progress: bool) -> None: cxx_toolchain_path(cxx_version, dir) -def validate_arm64_support(version: str) -> None: - """Raise if the requested CmdStan predates Windows ARM64 support.""" - if not is_windows() or determine_windows_arch() != 'aarch64': - return - if version.startswith('git:'): - return - try: - parsed = tuple( - int(part) for part in version.split('-')[0].split('.')[:2] - ) - except ValueError: - return - if parsed < MIN_WINDOWS_ARM64_VERSION: - minimum = '.'.join(str(part) for part in MIN_WINDOWS_ARM64_VERSION) - raise ValueError( - f'CmdStan {version} does not support Windows ARM64, ' - f'version {minimum} or later is required.' - ) - - def run_install(args: InteractiveSettings | InstallationSettings) -> None: """ Run a (potentially interactive) installation @@ -629,7 +605,6 @@ def run_install(args: InteractiveSettings | InstallationSettings) -> None: 'Connection to GitHub failed. ' 'Check firewall settings or ensure this version exists.' ) - validate_arm64_support(args.version) shutil.rmtree(cmdstan_version, ignore_errors=True) retrieve_version(args.version, args.progress) install_version( diff --git a/docsrc/installation.rst b/docsrc/installation.rst index c63c909c..7043af74 100644 --- a/docsrc/installation.rst +++ b/docsrc/installation.rst @@ -226,7 +226,7 @@ Windows on ARM64 CmdStan can be built on ARM 64-bit (``aarch64``) Windows machines using the ARM64 build of RTools 4.4 or 4.5, which CmdStanPy installs and detects -automatically. CmdStan 2.35 or later is required. +automatically. A Python interpreter built for Intel/AMD and running under emulation reports an ``x86_64`` architecture; in that case the Intel/AMD toolchain is used. The two diff --git a/test/test_cxx_installation.py b/test/test_cxx_installation.py index d508f58e..bdf8b1fb 100644 --- a/test/test_cxx_installation.py +++ b/test/test_cxx_installation.py @@ -12,7 +12,6 @@ import pytest from cmdstanpy import install_cxx_toolchain -from cmdstanpy.install_cmdstan import validate_arm64_support from cmdstanpy.utils import cmdstan as cmdstan_utils from cmdstanpy.utils import cxx_toolchain_path, make_command @@ -475,29 +474,6 @@ def test_install_defaults_to_latest_version( ) -@mark_windows_only -@pytest.mark.parametrize( - 'arch,version,ok', - [ - ('aarch64', '2.34.1', False), - ('aarch64', '2.35.0', True), - ('aarch64', '2.36.0', True), - ('aarch64', 'git:develop', True), - # the floor only applies to Windows ARM64 - ('x86_64', '2.30.0', True), - ], -) -def test_validate_arm64_support( - set_arch: SetArch, arch: str, version: str, ok: bool -) -> None: - set_arch(arch) - if ok: - validate_arm64_support(version) - else: - with pytest.raises(ValueError, match='does not support Windows ARM64'): - validate_arm64_support(version) - - # --------------------------------------------------------------------------- # make resolution # --------------------------------------------------------------------------- From bdff1f3c1ee7d0138d9a935147a3d370581aeb3b Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Mon, 21 Sep 2026 13:34:30 +0100 Subject: [PATCH 11/14] Drop RTools 3.5 support --- cmdstanpy/install_cmdstan.py | 1 - cmdstanpy/install_cxx_toolchain.py | 12 +++++------- cmdstanpy/utils/cmdstan.py | 7 ------- docsrc/installation.rst | 2 +- test/test_cxx_installation.py | 14 +++++--------- 5 files changed, 11 insertions(+), 25 deletions(-) diff --git a/cmdstanpy/install_cmdstan.py b/cmdstanpy/install_cmdstan.py index f28f8c13..b200d86d 100644 --- a/cmdstanpy/install_cmdstan.py +++ b/cmdstanpy/install_cmdstan.py @@ -665,7 +665,6 @@ def parse_cmdline_args() -> dict[str, Any]: if is_windows(): # use compiler installed with install_cxx_toolchain # Install a new compiler if compiler not found - # Search order is RTools40, RTools35 parser.add_argument( '--compiler', '-c', diff --git a/cmdstanpy/install_cxx_toolchain.py b/cmdstanpy/install_cxx_toolchain.py index 743824a7..1af57888 100644 --- a/cmdstanpy/install_cxx_toolchain.py +++ b/cmdstanpy/install_cxx_toolchain.py @@ -2,7 +2,7 @@ """ Download and install a C++ toolchain. Currently implemented platforms (platform.system) - Windows: RTools 3.5, 4.0 (default on x86), 4.4, 4.5 (default on ARM64) + Windows: RTools 4.0, 4.4, 4.5 (default) Darwin (macOS): Not implemented Linux: Not implemented Optional command line arguments: @@ -56,7 +56,7 @@ def usage() -> None: """Print usage.""" print( """Arguments: - -v (--version) : RTools version: 3.5, 4.0, 4.4 or 4.5 + -v (--version) : RTools version: 4.0, 4.4 or 4.5 -d (--dir) : install directory -s (--silent) : install with /VERYSILENT instead of /SILENT for RTools -m (--no-make) : don't install mingw32-make (Windows RTools 4.0 only) @@ -263,8 +263,6 @@ def get_url(version: str, arch: str | None = None) -> str: legacy = { ('4.0', 'x86_64'): 'rtools40-x86_64.exe', ('4.0', 'i686'): 'rtools40-i686.exe', - ('3.5', 'x86_64'): 'Rtools35.exe', - ('3.5', 'i686'): 'Rtools35.exe', }.get((version, arch), '') if legacy: return f'https://cran.r-project.org/bin/windows/Rtools/{legacy}' @@ -307,8 +305,8 @@ def run_rtools_install(args: dict[str, Any]) -> None: if not url: raise ValueError( f'RTools {version} is not available for {arch}. ' - f'Supported: {", ".join(sorted(RTOOLS_INSTALLERS))}, 4.0, 3.5 ' - '(4.0 and 3.5 are x86 only).' + f'Supported: {", ".join(sorted(RTOOLS_INSTALLERS))}, 4.0 ' + '(4.0 is x86 only).' ) if 'verbose' in args: @@ -372,7 +370,7 @@ def parse_cmdline_args() -> dict[str, Any]: parser.add_argument( '--version', '-v', - help="RTools version (3.5, 4.0, 4.4, 4.5), defaults to latest", + help="RTools version (4.0, 4.4, 4.5), defaults to latest", ) parser.add_argument( '--dir', '-d', help="install directory, defaults to '~/.cmdstan" diff --git a/cmdstanpy/utils/cmdstan.py b/cmdstanpy/utils/cmdstan.py index d50dd51c..1297be8b 100644 --- a/cmdstanpy/utils/cmdstan.py +++ b/cmdstanpy/utils/cmdstan.py @@ -349,14 +349,9 @@ class RToolsLayout(NamedTuple): ), RToolsLayout('4.0', 'x86_64', ('mingw64', 'bin'), ('usr', 'bin'), ('g++',)), RToolsLayout('4.0', 'i686', ('mingw32', 'bin'), ('usr', 'bin'), ('g++',)), - RToolsLayout('3.5', 'x86_64', ('mingw_64', 'bin'), ('bin',), ('g++',)), - RToolsLayout('3.5', 'i686', ('mingw_32', 'bin'), ('bin',), ('g++',)), ) _RTOOLS_VERSION_ALIASES = { - '3': '3.5', - '35': '3.5', - '3.5': '3.5', '4': '4.0', '40': '4.0', '4.0': '4.0', @@ -384,8 +379,6 @@ class RToolsLayout(NamedTuple): 'RTools43', 'RTools42', 'RTools40', - 'RTools35', - 'RTools30', 'RTools', ) diff --git a/docsrc/installation.rst b/docsrc/installation.rst index 7043af74..dd74eef6 100644 --- a/docsrc/installation.rst +++ b/docsrc/installation.rst @@ -160,7 +160,7 @@ This can be installed along with CmdStan when you invoke the function :meth:`cmd with argument ``compiler=True``. RTools is available for both Intel/AMD 64-bit (``x86_64``) and ARM 64-bit (``aarch64``) machines; CmdStanPy selects the matching build automatically. -Older toolchains (RTools 4.0 and 3.5, Intel/AMD only) are still detected if already installed. +Older toolchains (RTools 4.0, Intel/AMD only) are still detected if already installed. .. _install-cmdstan-fun: diff --git a/test/test_cxx_installation.py b/test/test_cxx_installation.py index bdf8b1fb..37c440ed 100644 --- a/test/test_cxx_installation.py +++ b/test/test_cxx_installation.py @@ -19,7 +19,6 @@ # (rtools version, arch) -> (compiler subdir, tool subdir, compiler exe) LAYOUTS = { - ('3.5', 'x86_64'): (('mingw_64', 'bin'), ('bin',), 'g++'), ('4.0', 'x86_64'): (('mingw64', 'bin'), ('usr', 'bin'), 'g++'), ('4.4', 'x86_64'): ( ('x86_64-w64-mingw32.static.posix', 'bin'), @@ -107,15 +106,15 @@ def make_toolchain( ('4.0', '4.0'), ('4', '4.0'), ('40', '4.0'), - ('3.5', '3.5'), - ('35', '3.5'), - ('3', '3.5'), ('4.2', '4.2'), ('42', '4.2'), ('4.4', '4.4'), ('44', '4.4'), ('4.5', '4.5'), ('45', '4.5'), + # no longer aliased, passed through unchanged + ('3.5', '3.5'), + ('35', '35'), ], ) def test_normalize_version(given: str, expected: str) -> None: @@ -161,13 +160,10 @@ def test_toolchain_name() -> None: 'https://cran.r-project.org/bin/windows/Rtools/' 'rtools40-x86_64.exe', ), - ( - '3.5', - 'x86_64', - 'https://cran.r-project.org/bin/windows/Rtools/Rtools35.exe', - ), # no ARM64 builds exist before RTools 4.4 ('4.0', 'aarch64', ''), + # RTools before 4.0 is no longer supported + ('3.5', 'x86_64', ''), ('3.5', 'aarch64', ''), ], ) From fa1987a067ab624b28af37421d1297acee852f8f Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Mon, 21 Sep 2026 13:41:58 +0100 Subject: [PATCH 12/14] Remove un-needed exports from cmdstanpy.utils --- .github/workflows/windows-toolchain.yml | 5 +++-- cmdstanpy/install_cmdstan.py | 3 +-- cmdstanpy/install_cxx_toolchain.py | 6 ++---- cmdstanpy/utils/__init__.py | 14 -------------- test/test_cxx_installation.py | 3 ++- 5 files changed, 8 insertions(+), 23 deletions(-) diff --git a/.github/workflows/windows-toolchain.yml b/.github/workflows/windows-toolchain.yml index b5c67c52..66ea39e5 100644 --- a/.github/workflows/windows-toolchain.yml +++ b/.github/workflows/windows-toolchain.yml @@ -67,7 +67,7 @@ jobs: - name: Report detected architecture run: | - python -c "import platform; from cmdstanpy.utils import determine_windows_arch as d; print(platform.machine(), d()); assert d() == '${{ matrix.arch }}'" + python -c "import platform; from cmdstanpy.utils.cmdstan import determine_windows_arch as d; print(platform.machine(), d()); assert d() == '${{ matrix.arch }}'" # The runner images ship their own RTools; make sure discovery only sees # what this job installed. @@ -88,7 +88,8 @@ jobs: run: | import os, shutil, subprocess from cmdstanpy.install_cxx_toolchain import get_toolchain_version, is_installed - from cmdstanpy.utils import cxx_toolchain_path, make_command + from cmdstanpy.utils import cxx_toolchain_path + from cmdstanpy.utils.cmdstan import make_command install_dir = os.path.join(os.environ["RUNNER_TEMP"], "toolchain") version = "${{ matrix.rtools }}" diff --git a/cmdstanpy/install_cmdstan.py b/cmdstanpy/install_cmdstan.py index b200d86d..35b129dc 100644 --- a/cmdstanpy/install_cmdstan.py +++ b/cmdstanpy/install_cmdstan.py @@ -40,12 +40,11 @@ from cmdstanpy.utils import ( cmdstan_path, do_command, - make_command, pushd, validate_dir, wrap_url_progress_hook, ) -from cmdstanpy.utils.cmdstan import get_download_url +from cmdstanpy.utils.cmdstan import get_download_url, make_command from . import progress as progbar diff --git a/cmdstanpy/install_cxx_toolchain.py b/cmdstanpy/install_cxx_toolchain.py index 1af57888..24c69b89 100644 --- a/cmdstanpy/install_cxx_toolchain.py +++ b/cmdstanpy/install_cxx_toolchain.py @@ -25,14 +25,12 @@ from typing import Any from cmdstanpy import _DOT_CMDSTAN -from cmdstanpy.utils import ( +from cmdstanpy.utils import pushd, validate_dir, wrap_url_progress_hook +from cmdstanpy.utils.cmdstan import ( determine_windows_arch, normalize_rtools_version, - pushd, rtools_compiler, rtools_layouts, - validate_dir, - wrap_url_progress_hook, ) EXTENSION = '.exe' if platform.system() == 'Windows' else '' diff --git a/cmdstanpy/utils/__init__.py b/cmdstanpy/utils/__init__.py index e821a20f..b4e6c9ab 100644 --- a/cmdstanpy/utils/__init__.py +++ b/cmdstanpy/utils/__init__.py @@ -8,19 +8,12 @@ from .cmdstan import ( EXTENSION, - RTOOLS_LAYOUTS, - RToolsLayout, cmdstan_path, cmdstan_version, cmdstan_version_before, cxx_toolchain_path, - determine_windows_arch, get_latest_cmdstan, install_cmdstan, - make_command, - normalize_rtools_version, - rtools_compiler, - rtools_layouts, set_cmdstan_path, set_make_env, validate_cmdstan_path, @@ -105,8 +98,6 @@ def show_versions(output: bool = True) -> str: __all__ = [ 'EXTENSION', - 'RTOOLS_LAYOUTS', - 'RToolsLayout', 'SanitizedOrTmpFilePath', 'build_xarray_data', 'check_sampler_csv', @@ -115,20 +106,15 @@ def show_versions(output: bool = True) -> str: 'cmdstan_version_before', 'create_named_text_file', 'cxx_toolchain_path', - 'determine_windows_arch', 'do_command', 'flatten_chains', 'get_latest_cmdstan', 'get_logger', 'install_cmdstan', - 'make_command', - 'normalize_rtools_version', 'parse_rdump_value', 'pushd', 'read_metric', 'rload', - 'rtools_compiler', - 'rtools_layouts', 'set_cmdstan_path', 'set_make_env', 'show_versions', diff --git a/test/test_cxx_installation.py b/test/test_cxx_installation.py index 37c440ed..dbdbc68a 100644 --- a/test/test_cxx_installation.py +++ b/test/test_cxx_installation.py @@ -13,7 +13,8 @@ from cmdstanpy import install_cxx_toolchain from cmdstanpy.utils import cmdstan as cmdstan_utils -from cmdstanpy.utils import cxx_toolchain_path, make_command +from cmdstanpy.utils import cxx_toolchain_path +from cmdstanpy.utils.cmdstan import make_command SetArch = Callable[[str], None] From 0299c009bfb46af4cb837237e8cdb9a421c7fd65 Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Wed, 23 Sep 2026 10:11:35 +0100 Subject: [PATCH 13/14] Clean up some Windows tests --- .github/workflows/windows-toolchain.yml | 59 ++++++++++++++++++------- test/test_compilation.py | 12 ----- test/test_cxx_installation.py | 12 ----- test/test_model.py | 16 ++++--- 4 files changed, 53 insertions(+), 46 deletions(-) diff --git a/.github/workflows/windows-toolchain.yml b/.github/workflows/windows-toolchain.yml index 66ea39e5..89d51179 100644 --- a/.github/workflows/windows-toolchain.yml +++ b/.github/workflows/windows-toolchain.yml @@ -40,18 +40,23 @@ jobs: include: - os: windows-latest arch: x86_64 + machine: x86-64 rtools: "4.0" - os: windows-latest arch: x86_64 + machine: x86-64 rtools: "4.4" - os: windows-latest arch: x86_64 + machine: x86-64 rtools: "4.5" - os: windows-11-arm arch: aarch64 + machine: ARM64 rtools: "4.4" - os: windows-11-arm arch: aarch64 + machine: ARM64 rtools: "4.5" steps: - uses: actions/checkout@v7 @@ -66,8 +71,14 @@ jobs: python -m pip install . - name: Report detected architecture + shell: python run: | - python -c "import platform; from cmdstanpy.utils.cmdstan import determine_windows_arch as d; print(platform.machine(), d()); assert d() == '${{ matrix.arch }}'" + import platform + from cmdstanpy.utils.cmdstan import determine_windows_arch + + arch = determine_windows_arch() + print(platform.machine(), "->", arch) + assert arch == "${{ matrix.arch }}", arch # The runner images ship their own RTools; make sure discovery only sees # what this job installed. @@ -105,25 +116,41 @@ jobs: # CmdStan invokes g++; on ARM64 it is an alias for clang cxx = shutil.which("g++") assert cxx and cxx.startswith(compiler_path), cxx - subprocess.run([cxx, "--version"], check=True) make = make_command() print("make:", make, "->", shutil.which(make)) assert shutil.which(make), f"{make} not found on PATH" subprocess.run([make, "--version"], check=True) - - name: Compile a trivial C++ program - shell: python - run: | - import os, subprocess, textwrap - from cmdstanpy.utils import cxx_toolchain_path + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as f: + f.write(f"CXX_BIN={compiler_path}\n") + f.write(f"TOOL_BIN={tool_path}\n") - cxx_toolchain_path("${{ matrix.rtools }}", os.path.join(os.environ["RUNNER_TEMP"], "toolchain")) - with open("hello.cpp", "w") as f: - f.write(textwrap.dedent(""" - #include - #include - int main() { std::cout << "hello\\n"; return 0; } - """)) - subprocess.run(["g++", "-O0", "hello.cpp", "-o", "hello.exe"], check=True) - subprocess.run(["./hello.exe"], check=True) + - name: Compile and run a trivial C++ program + shell: pwsh + run: | + #Requires -Version 7.4 + $PSNativeCommandUseErrorActionPreference = $true + $cxx = Join-Path $env:CXX_BIN 'g++.exe' + + & $cxx --version + $triple = (& $cxx -dumpmachine).Trim() + Write-Host "target triple: $triple" + if (-not $triple.StartsWith('${{ matrix.arch }}')) { + throw "expected a ${{ matrix.arch }} target, got '$triple'" + } + + Set-Content hello.cpp @( + '#include ', + '#include ', + 'int main() { std::cout << "hello" << std::endl; return 0; }' + ) + & $cxx -O0 hello.cpp -o hello.exe + ./hello.exe + + # file(1) ships in RTools' usr/bin; proves what was actually emitted + $described = & (Join-Path $env:TOOL_BIN 'file.exe') hello.exe + Write-Host $described + if ($described -notlike '*${{ matrix.machine }}*') { + throw "expected a ${{ matrix.machine }} binary, got '$described'" + } diff --git a/test/test_compilation.py b/test/test_compilation.py index 4ef5b8c8..addaea55 100644 --- a/test/test_compilation.py +++ b/test/test_compilation.py @@ -201,18 +201,6 @@ def test_user_header() -> None: opts.validate() -def test_user_header_is_posix_path() -> None: - """Backslashes in the -include path are eaten by clang.""" - opts = CompilerOptions( - user_header=os.path.join(DATAFILES_PATH, 'return_one.hpp') - ) - opts.validate() - assert '\\' not in opts.user_header - assert opts.user_header.endswith('/return_one.hpp') - assert os.path.isfile(opts.user_header) - assert str(opts.cpp_options['USER_HEADER']) == opts.user_header - - def test_model_format_options() -> None: stan = os.path.join(DATAFILES_PATH, 'format_me.stan') diff --git a/test/test_cxx_installation.py b/test/test_cxx_installation.py index dbdbc68a..ebe901f7 100644 --- a/test/test_cxx_installation.py +++ b/test/test_cxx_installation.py @@ -4,7 +4,6 @@ import os import platform -import sys from pathlib import Path from test import mark_not_windows, mark_windows_only from typing import Callable @@ -514,14 +513,3 @@ def test_make_command_windows( def test_usage(capsys: pytest.CaptureFixture) -> None: install_cxx_toolchain.usage() assert '--version' in capsys.readouterr().out - - -def test_parse_cmdline_args(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - sys, 'argv', ['install_cxx_toolchain', '-v', '4.5', '-d', 'somewhere'] - ) - args = install_cxx_toolchain.parse_cmdline_args() - assert args['version'] == '4.5' - assert args['dir'] == 'somewhere' - assert args['silent'] is False - assert args['progress'] is False diff --git a/test/test_model.py b/test/test_model.py index b9f4a3ce..f2292b14 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -43,12 +43,6 @@ def test_model_good() -> None: assert os.path.samefile(model.exe_file, BERN_EXE) assert 'bernoulli' == model.name - # compile with external header - model = CmdStanModel( - stan_file=os.path.join(DATAFILES_PATH, "external.stan"), - user_header=os.path.join(DATAFILES_PATH, 'return_one.hpp'), - ) - # default model name model = CmdStanModel(stan_file=BERN_STAN) assert BERN_BASENAME == model.name @@ -59,6 +53,16 @@ def test_model_good() -> None: assert os.path.samefile(model.exe_file, BERN_EXE) +def test_model_compile_user_header() -> None: + """The header path is passed with this platform's native separators.""" + model = CmdStanModel( + stan_file=os.path.join(DATAFILES_PATH, "external.stan"), + user_header=os.path.join(DATAFILES_PATH, 'return_one.hpp'), + force_compile=True, + ) + assert os.path.exists(model.exe_file) + + def test_ctor_compile_arg() -> None: if os.path.exists(BERN_EXE): os.remove(BERN_EXE) From c86a7adba84bcbd54a35a088dcc659c6bf3f24c2 Mon Sep 17 00:00:00 2001 From: Nikhil Dabas Date: Wed, 23 Sep 2026 11:12:21 +0100 Subject: [PATCH 14/14] Clarify why path normalization is needed --- cmdstanpy/compilation.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmdstanpy/compilation.py b/cmdstanpy/compilation.py index 21484c9a..b02697b3 100644 --- a/cmdstanpy/compilation.py +++ b/cmdstanpy/compilation.py @@ -200,8 +200,9 @@ def validate_user_header(self) -> None: if "allow-undefined" not in self._stanc_options: self._stanc_options["allow-undefined"] = True # CmdStan's make/program does not apply its usual - # $(subst \,/,...) to USER_HEADER, and clang reads the remaining - # backslashes in -include as escapes + # $(subst \,/,...) to USER_HEADER, and Stan Math sets SHELL on + # Windows, so recipes run through sh, which eats the backslashes + # before the compiler sees the -include path self._user_header = Path(self._user_header).absolute().as_posix() if ' ' in self._user_header: