diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7c14cc7e..8d16fce2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,8 +43,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest, windows-11-arm] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + exclude: + # CPython has no Windows ARM64 release before 3.11 + - os: windows-11-arm + python-version: "3.10" env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} steps: @@ -80,21 +84,21 @@ jobs: uses: actions/cache@v6 with: path: ~/.cmdstan - key: ${{ runner.os }}-cmdstan-${{ needs.get-cmdstan-version.outputs.version }}-${{ hashFiles('**/install_cmdstan.py') }} + key: ${{ runner.os }}-${{ runner.arch }}-cmdstan-${{ needs.get-cmdstan-version.outputs.version }}-${{ hashFiles('**/install_cmdstan.py') }} - name: Delete precompiled header (MacOS) if: matrix.os == 'macos-latest' && steps.cache-cmdstan.outputs.cache-hit == 'true' run: rm -rf ~/.cmdstan/cmdstan-${{ needs.get-cmdstan-version.outputs.version }}/stan/src/stan/model/*.hpp.gch - name: Install CmdStan (Linux, macOS) - if: matrix.os != 'windows-latest' + if: runner.os != 'Windows' run: | install_cmdstan -h install_cxx_toolchain -h python -c "import cmdstanpy; cmdstanpy.install_cmdstan(version='${{ needs.get-cmdstan-version.outputs.version }}', cores=4)" - name: Install CmdStan (Windows) - if: matrix.os == 'windows-latest' + if: runner.os == 'Windows' run: | install_cmdstan -h install_cxx_toolchain -h diff --git a/.github/workflows/windows-toolchain.yml b/.github/workflows/windows-toolchain.yml new file mode 100644 index 00000000..89d51179 --- /dev/null +++ b/.github/workflows/windows-toolchain.yml @@ -0,0 +1,156 @@ +name: Windows toolchain + +# Exercises the RTools install + discovery path on real Windows runners. +# Kept out of the main matrix because the RTools 4.4/4.5 installers are ~450MB. + +on: + workflow_dispatch: + pull_request: + paths: + - "cmdstanpy/install_cxx_toolchain.py" + - "cmdstanpy/utils/cmdstan.py" + - "cmdstanpy/install_cmdstan.py" + - "test/test_cxx_installation.py" + - ".github/workflows/windows-toolchain.yml" + +jobs: + # Fast, hardware-independent: the layout table is exercised with fake trees. + unit: + name: layout unit tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-latest, windows-11-arm] + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - run: python -m pip install --upgrade pip + - run: python -m pip install .[test] + - run: python -m pytest test/test_cxx_installation.py -v + + install: + name: RTools ${{ matrix.rtools }} on ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: windows-latest + arch: x86_64 + machine: x86-64 + rtools: "4.0" + - os: windows-latest + arch: x86_64 + machine: x86-64 + rtools: "4.4" + - os: windows-latest + arch: x86_64 + machine: x86-64 + rtools: "4.5" + - os: windows-11-arm + arch: aarch64 + machine: ARM64 + rtools: "4.4" + - os: windows-11-arm + arch: aarch64 + machine: ARM64 + rtools: "4.5" + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Install package + run: | + python -m pip install --upgrade pip + python -m pip install . + + - name: Report detected architecture + shell: python + run: | + import platform + from cmdstanpy.utils.cmdstan import determine_windows_arch + + arch = determine_windows_arch() + print(platform.machine(), "->", arch) + assert arch == "${{ matrix.arch }}", arch + + # The runner images ship their own RTools; make sure discovery only sees + # what this job installed. + - name: Clear pre-existing RTools environment + run: | + "RTOOLS40_HOME=" >> $env:GITHUB_ENV + "RTOOLS42_HOME=" >> $env:GITHUB_ENV + "RTOOLS43_HOME=" >> $env:GITHUB_ENV + "RTOOLS44_HOME=" >> $env:GITHUB_ENV + "RTOOLS45_HOME=" >> $env:GITHUB_ENV + + - name: Install RTools ${{ matrix.rtools }} + run: | + python -m cmdstanpy.install_cxx_toolchain --version ${{ matrix.rtools }} --dir ${{ runner.temp }}\toolchain --verbose + + - name: Discover and activate toolchain + shell: python + run: | + import os, shutil, subprocess + from cmdstanpy.install_cxx_toolchain import get_toolchain_version, is_installed + from cmdstanpy.utils import cxx_toolchain_path + from cmdstanpy.utils.cmdstan import make_command + + install_dir = os.path.join(os.environ["RUNNER_TEMP"], "toolchain") + version = "${{ matrix.rtools }}" + folder = get_toolchain_version("RTools", version) + print("expecting install folder:", folder) + assert is_installed(os.path.join(install_dir, folder), version) + + compiler_path, tool_path = cxx_toolchain_path(version, install_dir) + print("compiler:", compiler_path) + print("tools: ", tool_path) + assert compiler_path.startswith(install_dir), compiler_path + + # CmdStan invokes g++; on ARM64 it is an alias for clang + cxx = shutil.which("g++") + assert cxx and cxx.startswith(compiler_path), cxx + + make = make_command() + print("make:", make, "->", shutil.which(make)) + assert shutil.which(make), f"{make} not found on PATH" + subprocess.run([make, "--version"], check=True) + + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as f: + f.write(f"CXX_BIN={compiler_path}\n") + f.write(f"TOOL_BIN={tool_path}\n") + + - name: Compile and run a trivial C++ program + shell: pwsh + run: | + #Requires -Version 7.4 + $PSNativeCommandUseErrorActionPreference = $true + $cxx = Join-Path $env:CXX_BIN 'g++.exe' + + & $cxx --version + $triple = (& $cxx -dumpmachine).Trim() + Write-Host "target triple: $triple" + if (-not $triple.StartsWith('${{ matrix.arch }}')) { + throw "expected a ${{ matrix.arch }} target, got '$triple'" + } + + Set-Content hello.cpp @( + '#include ', + '#include ', + 'int main() { std::cout << "hello" << std::endl; return 0; }' + ) + & $cxx -O0 hello.cpp -o hello.exe + ./hello.exe + + # file(1) ships in RTools' usr/bin; proves what was actually emitted + $described = & (Join-Path $env:TOOL_BIN 'file.exe') hello.exe + Write-Host $described + if ($described -notlike '*${{ matrix.machine }}*') { + throw "expected a ${{ matrix.machine }} binary, got '$described'" + } diff --git a/cmdstanpy/compilation.py b/cmdstanpy/compilation.py index afc75812..b02697b3 100644 --- a/cmdstanpy/compilation.py +++ b/cmdstanpy/compilation.py @@ -5,7 +5,6 @@ import io import json import os -import platform import shutil import subprocess from datetime import datetime @@ -13,7 +12,12 @@ from typing import Any, Iterable from cmdstanpy.utils import get_logger -from cmdstanpy.utils.cmdstan import EXTENSION, cmdstan_path, stanc_path +from cmdstanpy.utils.cmdstan import ( + EXTENSION, + cmdstan_path, + make_command, + stanc_path, +) from cmdstanpy.utils.command import do_command from cmdstanpy.utils.filesystem import SanitizedOrTmpFilePath @@ -195,8 +199,11 @@ def validate_user_header(self) -> None: ) if "allow-undefined" not in self._stanc_options: self._stanc_options["allow-undefined"] = True - # set full path - self._user_header = os.path.abspath(self._user_header) + # CmdStan's make/program does not apply its usual + # $(subst \,/,...) to USER_HEADER, and Stan Math sets SHELL on + # Windows, so recipes run through sh, which eats the backslashes + # before the compiler sees the -include path + self._user_header = Path(self._user_header).absolute().as_posix() if ' ' in self._user_header: raise ValueError( @@ -372,10 +379,7 @@ def compile_stan_file( exe_target, ) - make = os.getenv( - 'MAKE', - 'make' if platform.system() != 'Windows' else 'mingw32-make', - ) + make = make_command() cmd = [make] cmd.extend(compiler_options.compose(filename_in_msg=src.name)) cmd.append(Path(exe_file).as_posix()) diff --git a/cmdstanpy/install_cmdstan.py b/cmdstanpy/install_cmdstan.py index 06be41f9..35b129dc 100644 --- a/cmdstanpy/install_cmdstan.py +++ b/cmdstanpy/install_cmdstan.py @@ -44,7 +44,7 @@ validate_dir, wrap_url_progress_hook, ) -from cmdstanpy.utils.cmdstan import get_download_url +from cmdstanpy.utils.cmdstan import get_download_url, make_command from . import progress as progbar @@ -71,7 +71,6 @@ def is_windows() -> bool: return platform.system() == 'Windows' -MAKE = os.getenv('MAKE', 'make' if not is_windows() else 'mingw32-make') EXTENSION = '.exe' if is_windows() else '' @@ -203,7 +202,7 @@ def overwrite(self) -> bool: def compiler(self) -> bool: if not is_windows(): return False - print("Would you like to install the RTools40 C++ toolchain?") + print("Would you like to install the RTools C++ toolchain?") print("A C++ toolchain is required for CmdStan.") print( "If you are not sure if you need the toolchain or not, " @@ -234,7 +233,7 @@ def clean_all(verbose: bool = False) -> None: :param verbose: Boolean value; when ``True``, show output from make command. """ - cmd = [MAKE, 'clean-all'] + cmd = [make_command(), 'clean-all'] try: if verbose: do_command(cmd) @@ -262,7 +261,7 @@ def build(verbose: bool = False, progress: bool = True, cores: int = 1) -> None: :param cores: Integer, number of cores to use in the ``make`` command. Default is 1 core. """ - cmd = [MAKE, 'build', f'-j{cores}'] + cmd = [make_command(), 'build', f'-j{cores}'] try: if verbose: do_command(cmd) @@ -343,7 +342,7 @@ def compile_example(verbose: bool = False) -> None: if path.is_file(): path.unlink() - cmd = [MAKE, path.as_posix()] + cmd = [make_command(), path.as_posix()] try: if verbose: do_command(cmd) @@ -544,37 +543,26 @@ def retrieve_version(version: str, progress: bool = True) -> None: def run_compiler_install(dir: str, verbose: bool, progress: bool) -> None: - from .install_cxx_toolchain import is_installed as _is_installed_cxx + from .install_cxx_toolchain import latest_version as _latest_version_cxx from .install_cxx_toolchain import run_rtools_install as _main_cxx from .utils import cxx_toolchain_path - compiler_found = False - rtools40_home = os.environ.get('RTOOLS40_HOME') - for cxx_loc in ([rtools40_home] if rtools40_home is not None else []) + [ - home_cmdstan(), - os.path.join(os.path.abspath("/"), "RTools40"), - os.path.join(os.path.abspath("/"), "RTools"), - os.path.join(os.path.abspath("/"), "RTools35"), - os.path.join(os.path.abspath("/"), "RBuildTools"), - ]: - for cxx_version in ['40', '35']: - if _is_installed_cxx(cxx_loc, cxx_version): - compiler_found = True - break - if compiler_found: - break - if not compiler_found: - print('Installing RTools40') - # copy argv and clear sys.argv - _main_cxx( - { - 'dir': dir, - 'progress': progress, - 'version': None, - 'verbose': verbose, - } - ) - cxx_version = '40' + try: + cxx_toolchain_path(None, dir) + return + except ValueError: + pass + + cxx_version = _latest_version_cxx() + print(f'Installing RTools {cxx_version}') + _main_cxx( + { + 'dir': dir, + 'progress': progress, + 'version': None, + 'verbose': verbose, + } + ) # Add toolchain to $PATH cxx_toolchain_path(cxx_version, dir) @@ -676,7 +664,6 @@ def parse_cmdline_args() -> dict[str, Any]: if is_windows(): # use compiler installed with install_cxx_toolchain # Install a new compiler if compiler not found - # Search order is RTools40, RTools35 parser.add_argument( '--compiler', '-c', diff --git a/cmdstanpy/install_cxx_toolchain.py b/cmdstanpy/install_cxx_toolchain.py index 4fbf86b9..24c69b89 100644 --- a/cmdstanpy/install_cxx_toolchain.py +++ b/cmdstanpy/install_cxx_toolchain.py @@ -2,7 +2,7 @@ """ Download and install a C++ toolchain. Currently implemented platforms (platform.system) - Windows: RTools 3.5, 4.0 (default) + Windows: RTools 4.0, 4.4, 4.5 (default) Darwin (macOS): Not implemented Linux: Not implemented Optional command line arguments: @@ -26,16 +26,35 @@ from cmdstanpy import _DOT_CMDSTAN from cmdstanpy.utils import pushd, validate_dir, wrap_url_progress_hook +from cmdstanpy.utils.cmdstan import ( + determine_windows_arch, + normalize_rtools_version, + rtools_compiler, + rtools_layouts, +) EXTENSION = '.exe' if platform.system() == 'Windows' else '' -IS_64BITS = sys.maxsize > 2**32 + +# CRAN embeds build revisions in the RTools 4.2+ installer filenames, so we +# use the r-hub mirror, which publishes them under a stable 'latest' tag. +# These are the builds the CmdStan guide points users at. +RTOOLS_INSTALLERS = { + '4.5': { + 'x86_64': 'rtools45.exe', + 'aarch64': 'rtools45-aarch64.exe', + }, + '4.4': { + 'x86_64': 'rtools44.exe', + 'aarch64': 'rtools44-aarch64.exe', + }, +} def usage() -> None: """Print usage.""" print( """Arguments: - -v (--version) :CmdStan version + -v (--version) : RTools version: 4.0, 4.4 or 4.5 -d (--dir) : install directory -s (--silent) : install with /VERYSILENT instead of /SILENT for RTools -m (--no-make) : don't install mingw32-make (Windows RTools 4.0 only) @@ -111,13 +130,14 @@ def install_version( def install_mingw32_make(toolchain_loc: str, verbose: bool = False) -> None: """Install mingw32-make for Windows RTools 4.0.""" + arch = determine_windows_arch() os.environ['PATH'] = ';'.join( list( OrderedDict.fromkeys( [ os.path.join( toolchain_loc, - 'mingw_64' if IS_64BITS else 'mingw_32', + 'mingw64' if arch == 'x86_64' else 'mingw32', 'bin', ), os.path.join(toolchain_loc, 'usr', 'bin'), @@ -129,7 +149,11 @@ def install_mingw32_make(toolchain_loc: str, verbose: bool = False) -> None: cmd = [ 'pacman', '-Sy', - 'mingw-w64-x86_64-make' if IS_64BITS else 'mingw-w64-i686-make', + ( + 'mingw-w64-x86_64-make' + if arch == 'x86_64' + else 'mingw-w64-i686-make' + ), '--noconfirm', ] with pushd('.'): @@ -162,37 +186,22 @@ def install_mingw32_make(toolchain_loc: str, verbose: bool = False) -> None: def is_installed(toolchain_loc: str, version: str) -> bool: """Returns True is toolchain is installed.""" - if platform.system() == 'Windows': - if version in ['35', '3.5']: - if not os.path.exists(os.path.join(toolchain_loc, 'bin')): - return False - return os.path.exists( - os.path.join( - toolchain_loc, - 'mingw_64' if IS_64BITS else 'mingw_32', - 'bin', - 'g++' + EXTENSION, - ) - ) - elif version in ['40', '4.0', '4']: - return os.path.exists( - os.path.join( - toolchain_loc, - 'mingw64' if IS_64BITS else 'mingw32', - 'bin', - 'g++' + EXTENSION, - ) - ) - else: - return False + if platform.system() != 'Windows': + return False + for layout in rtools_layouts(normalize_rtools_version(version)): + tool_path = os.path.join(toolchain_loc, *layout.tool_subdir) + if not os.path.exists(tool_path): + continue + if rtools_compiler(toolchain_loc, layout) is not None: + return True return False def latest_version() -> str: - """Windows version hardcoded to 4.0.""" - if platform.system() == 'Windows': - return '4.0' - return '' + """Latest RTools version supported on this machine.""" + if platform.system() != 'Windows': + return '' + return '4.5' def retrieve_toolchain(filename: str, url: str, progress: bool = True) -> None: @@ -222,10 +231,7 @@ def retrieve_toolchain(filename: str, url: str, progress: bool = True) -> None: def normalize_version(version: str) -> str: """Return maj.min part of version string.""" if platform.system() == 'Windows': - if version in ['4', '40']: - version = '4.0' - elif version == '35': - version = '3.5' + return normalize_rtools_version(version) return version @@ -237,28 +243,42 @@ def get_toolchain_name() -> str: # TODO(2.0): consider something other than RTools -def get_url(version: str) -> str: +def get_url(version: str, arch: str | None = None) -> str: """Return URL for toolchain.""" - url = '' - if platform.system() == 'Windows': - if version == '4.0': - # pylint: disable=line-too-long - if IS_64BITS: - url = 'https://cran.r-project.org/bin/windows/Rtools/rtools40-x86_64.exe' # noqa: disable=E501 - else: - url = 'https://cran.r-project.org/bin/windows/Rtools/rtools40-i686.exe' # noqa: disable=E501 - elif version == '3.5': - url = 'https://cran.r-project.org/bin/windows/Rtools/Rtools35.exe' - return url - + if platform.system() != 'Windows': + return '' + if arch is None: + arch = determine_windows_arch() + if version in RTOOLS_INSTALLERS: + installer = RTOOLS_INSTALLERS[version].get(arch, '') + if not installer: + return '' + series = 'rtools' + version.replace('.', '') + return ( + f'https://github.com/r-hub/{series}/releases/' + f'download/latest/{installer}' + ) + legacy = { + ('4.0', 'x86_64'): 'rtools40-x86_64.exe', + ('4.0', 'i686'): 'rtools40-i686.exe', + }.get((version, arch), '') + if legacy: + return f'https://cran.r-project.org/bin/windows/Rtools/{legacy}' + return '' -def get_toolchain_version(name: str, version: str) -> str: - """Toolchain version.""" - toolchain_folder = '' - if platform.system() == 'Windows': - toolchain_folder = '{}{}'.format(name, version.replace('.', '')) - return toolchain_folder +def get_toolchain_version( + name: str, version: str, arch: str | None = None +) -> str: + """Toolchain install folder name.""" + if platform.system() != 'Windows': + return '' + if arch is None: + arch = determine_windows_arch() + folder = '{}{}'.format(name, version.replace('.', '')) + if arch == 'aarch64': + folder += '-aarch64' + return folder def run_rtools_install(args: dict[str, Any]) -> None: @@ -274,9 +294,18 @@ def run_rtools_install(args: dict[str, Any]) -> None: if version is None: version = latest_version() version = normalize_version(version) - print("C++ toolchain '{}' version: {}".format(toolchain, version)) + arch = determine_windows_arch() + print( + "C++ toolchain '{}' version: {} ({})".format(toolchain, version, arch) + ) - url = get_url(version) + url = get_url(version, arch) + if not url: + raise ValueError( + f'RTools {version} is not available for {arch}. ' + f'Supported: {", ".join(sorted(RTOOLS_INSTALLERS))}, 4.0 ' + '(4.0 is x86 only).' + ) if 'verbose' in args: verbose = args['verbose'] @@ -297,12 +326,12 @@ def run_rtools_install(args: dict[str, Any]) -> None: if platform.system() == 'Windows': silent = 'silent' in args # force silent == False for 4.0 version - if 'silent' not in args and version in ('4.0', '4', '40'): + if 'silent' not in args and version == '4.0': silent = False else: silent = False - toolchain_folder = get_toolchain_version(toolchain, version) + toolchain_folder = get_toolchain_version(toolchain, version, arch) with pushd(install_dir): if is_installed(toolchain_folder, version): print('C++ toolchain {} already installed'.format(toolchain_folder)) @@ -322,7 +351,7 @@ def run_rtools_install(args: dict[str, Any]) -> None: if ( 'no-make' not in args and (platform.system() == 'Windows') - and (version in ('4.0', '4', '40')) + and version == '4.0' ): if os.path.exists( os.path.join( @@ -336,7 +365,11 @@ def run_rtools_install(args: dict[str, Any]) -> None: def parse_cmdline_args() -> dict[str, Any]: parser = argparse.ArgumentParser() - parser.add_argument('--version', '-v', help="version, defaults to latest") + parser.add_argument( + '--version', + '-v', + help="RTools version (4.0, 4.4, 4.5), defaults to latest", + ) parser.add_argument( '--dir', '-d', help="install directory, defaults to '~/.cmdstan" ) diff --git a/cmdstanpy/utils/cmdstan.py b/cmdstanpy/utils/cmdstan.py index 77c1894d..1297be8b 100644 --- a/cmdstanpy/utils/cmdstan.py +++ b/cmdstanpy/utils/cmdstan.py @@ -2,12 +2,14 @@ Utilities for finding and installing CmdStan """ +import logging import os import platform +import shutil import subprocess import sys from collections import OrderedDict -from typing import Callable +from typing import Callable, NamedTuple from tqdm.auto import tqdm @@ -20,6 +22,24 @@ EXTENSION = '.exe' if platform.system() == 'Windows' else '' +def determine_windows_arch() -> str: + """ + Return the architecture of the running Windows process: + ``'x86_64'``, ``'aarch64'`` or ``'i686'``. + + A CPython built for x86-64 running under emulation on an ARM64 machine + reports ``AMD64``, which is what we want: it needs the Intel RTools. + """ + machine = platform.machine().upper() + if machine == 'ARM64': + return 'aarch64' + if machine in ('AMD64', 'X86_64'): + return 'x86_64' + if machine in ('X86', 'I386', 'I686'): + return 'i686' + return 'x86_64' if sys.maxsize > 2**32 else 'i686' + + def determine_linux_arch() -> str: machine = platform.machine() arch = "" @@ -271,6 +291,220 @@ def cmdstan_version_before( return False +class RToolsLayout(NamedTuple): + """Directory layout of one RTools version/architecture combination.""" + + version: str + arch: str + compiler_subdir: tuple[str, ...] + tool_subdir: tuple[str, ...] + compilers: tuple[str, ...] + + +# Ordered newest-first so that a search without an explicit version +# prefers the most recent layout. RTools 4.2 through 4.5 share a layout. +# The ARM builds are LLVM-based but ship gcc/g++ aliases for clang. +RTOOLS_LAYOUTS = ( + RToolsLayout( + '4.5', + 'aarch64', + ('aarch64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + ('g++', 'clang++'), + ), + RToolsLayout( + '4.5', + 'x86_64', + ('x86_64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + ('g++',), + ), + RToolsLayout( + '4.4', + 'aarch64', + ('aarch64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + ('g++', 'clang++'), + ), + RToolsLayout( + '4.4', + 'x86_64', + ('x86_64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + ('g++',), + ), + RToolsLayout( + '4.3', + 'x86_64', + ('x86_64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + ('g++',), + ), + RToolsLayout( + '4.2', + 'x86_64', + ('x86_64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + ('g++',), + ), + RToolsLayout('4.0', 'x86_64', ('mingw64', 'bin'), ('usr', 'bin'), ('g++',)), + RToolsLayout('4.0', 'i686', ('mingw32', 'bin'), ('usr', 'bin'), ('g++',)), +) + +_RTOOLS_VERSION_ALIASES = { + '4': '4.0', + '40': '4.0', + '4.0': '4.0', + '42': '4.2', + '4.2': '4.2', + '43': '4.3', + '4.3': '4.3', + '44': '4.4', + '4.4': '4.4', + '45': '4.5', + '4.5': '4.5', +} + +_RTOOLS_HOME_VARS = ( + 'RTOOLS45_HOME', + 'RTOOLS44_HOME', + 'RTOOLS43_HOME', + 'RTOOLS42_HOME', + 'RTOOLS40_HOME', +) + +_RTOOLS_DIR_NAMES = ( + 'RTools45', + 'RTools44', + 'RTools43', + 'RTools42', + 'RTools40', + 'RTools', +) + + +def normalize_rtools_version(version: str) -> str: + """Return the ``maj.min`` form of an RTools version string.""" + return _RTOOLS_VERSION_ALIASES.get(version, version) + + +def rtools_layouts( + version: str | None = None, arch: str | None = None +) -> list[RToolsLayout]: + """ + Return the known RTools directory layouts matching a version and + architecture, newest first. An empty list means the combination is + not supported. + """ + if arch is None: + arch = determine_windows_arch() + return [ + layout + for layout in RTOOLS_LAYOUTS + if layout.arch == arch + and (version is None or layout.version == version) + ] + + +def _rtools_dir_names(arch: str) -> list[str]: + names = [] + for name in _RTOOLS_DIR_NAMES: + if arch == 'aarch64': + # CRAN installs the ARM builds to e.g. C:\rtools45-aarch64 + names.append(f'{name}-aarch64') + names.append(name) + return names + + +def _rtools_search_roots(install_dir: str | None, arch: str) -> list[str]: + """Candidate RTools installation roots, in order of preference.""" + roots = [ + home + for home in (os.environ.get(var) for var in _RTOOLS_HOME_VARS) + if home + ] + names = _rtools_dir_names(arch) + parents = [] + if install_dir is not None: + parents.append(install_dir) + parents.append(os.path.expanduser(os.path.join('~', _DOT_CMDSTAN))) + parents.append(os.path.abspath('/')) + for parent in parents: + roots.extend(os.path.join(parent, name) for name in names) + roots.append(os.path.join(os.path.abspath('/'), 'RBuildTools')) + return roots + + +def rtools_compiler(toolchain_root: str, layout: RToolsLayout) -> str | None: + """Return the name of the C++ compiler shipped in an installation.""" + compiler_dir = os.path.join(toolchain_root, *layout.compiler_subdir) + for compiler in layout.compilers: + if os.path.exists(os.path.join(compiler_dir, compiler + EXTENSION)): + return compiler + return None + + +def _probe_rtools_root( + toolchain_root: str, + layouts: list[RToolsLayout], + logger: logging.Logger, +) -> tuple[str, str, str] | None: + """ + Match an installation root against the given layouts, returning + ``(root, compiler_path, tool_path)`` or ``None``. + """ + if not toolchain_root or not os.path.exists(toolchain_root): + return None + for layout in layouts: + # RTools 4.2+ ship empty mingw64/ucrt64/clangarm64 stub directories, + # so the compiler binary itself must be checked, not just the dir + if rtools_compiler(toolchain_root, layout) is None: + continue + compiler_path = os.path.join(toolchain_root, *layout.compiler_subdir) + tool_path = os.path.join(toolchain_root, *layout.tool_subdir) + if os.path.exists(tool_path): + return toolchain_root, compiler_path, tool_path + logger.warning( + 'Found invalid RTools installation on %s: missing %s', + toolchain_root, + tool_path, + ) + return None + logger.warning('Found no usable RTools installation on %s', toolchain_root) + return None + + +def make_command() -> str: + """ + Name of the GNU Make executable to use. + + RTools 4.0 ships ``mingw32-make``, while RTools 4.2 and later ship + plain ``make`` in ``usr/bin``. If neither is on the ``$PATH``, an RTools + installation managed by CmdStanPy is activated before giving up. + """ + make = os.environ.get('MAKE') + if make: + return make + if platform.system() != 'Windows': + return 'make' + + def _found() -> str | None: + for candidate in ('mingw32-make', 'make'): + if shutil.which(candidate): + return candidate + return None + + found = _found() + if found is None: + try: + cxx_toolchain_path() + except ValueError: + pass + else: + found = _found() + return found or 'make' + + def cxx_toolchain_path( version: str | None = None, install_dir: str | None = None ) -> tuple[str, ...]: @@ -284,143 +518,33 @@ def cxx_toolchain_path( if version is not None and not isinstance(version, str): raise TypeError('Format version number as a string') logger = get_logger() - if 'CMDSTAN_TOOLCHAIN' in os.environ: - toolchain_root = os.environ['CMDSTAN_TOOLCHAIN'] - if os.path.exists(os.path.join(toolchain_root, 'mingw64')): - compiler_path = os.path.join( - toolchain_root, - 'mingw64' if (sys.maxsize > 2**32) else 'mingw32', - 'bin', - ) - if os.path.exists(compiler_path): - tool_path = os.path.join(toolchain_root, 'usr', 'bin') - if not os.path.exists(tool_path): - tool_path = '' - compiler_path = '' - logger.warning( - 'Found invalid installion for RTools40 on %s', - toolchain_root, - ) - toolchain_root = '' - else: - compiler_path = '' - logger.warning( - 'Found invalid installion for RTools40 on %s', - toolchain_root, - ) - toolchain_root = '' - elif os.path.exists(os.path.join(toolchain_root, 'mingw_64')): - compiler_path = os.path.join( - toolchain_root, - 'mingw_64' if (sys.maxsize > 2**32) else 'mingw_32', - 'bin', - ) - if os.path.exists(compiler_path): - tool_path = os.path.join(toolchain_root, 'bin') - if not os.path.exists(tool_path): - tool_path = '' - compiler_path = '' - logger.warning( - 'Found invalid installion for RTools35 on %s', - toolchain_root, - ) - toolchain_root = '' - else: - compiler_path = '' - logger.warning( - 'Found invalid installion for RTools35 on %s', - toolchain_root, - ) - toolchain_root = '' + arch = determine_windows_arch() + layouts = rtools_layouts( + normalize_rtools_version(version) if version else None, arch + ) + if not layouts: + raise ValueError(f'unsupported RTools version: {version}') + + found = None + if 'CMDSTAN_TOOLCHAIN' in os.environ: + found = _probe_rtools_root( + os.environ['CMDSTAN_TOOLCHAIN'], layouts, logger + ) else: - rtools40_home = os.environ.get('RTOOLS40_HOME') - cmdstan_dir = os.path.expanduser(os.path.join('~', _DOT_CMDSTAN)) - for toolchain_root in ( - ([rtools40_home] if rtools40_home is not None else []) - + ( - [ - os.path.join(install_dir, 'RTools40'), - os.path.join(install_dir, 'RTools35'), - os.path.join(install_dir, 'RTools30'), - os.path.join(install_dir, 'RTools'), - ] - if install_dir is not None - else [] - ) - + [ - os.path.join(cmdstan_dir, 'RTools40'), - os.path.join(os.path.abspath("/"), "RTools40"), - os.path.join(cmdstan_dir, 'RTools35'), - os.path.join(os.path.abspath("/"), "RTools35"), - os.path.join(cmdstan_dir, 'RTools'), - os.path.join(os.path.abspath("/"), "RTools"), - os.path.join(os.path.abspath("/"), "RBuildTools"), - ] - ): - compiler_path = '' - tool_path = '' - - if os.path.exists(toolchain_root): - if version not in ('35', '3.5', '3'): - compiler_path = os.path.join( - toolchain_root, - 'mingw64' if (sys.maxsize > 2**32) else 'mingw32', - 'bin', - ) - if os.path.exists(compiler_path): - tool_path = os.path.join(toolchain_root, 'usr', 'bin') - if not os.path.exists(tool_path): - tool_path = '' - compiler_path = '' - logger.warning( - 'Found invalid installation for RTools40 on %s', - toolchain_root, - ) - toolchain_root = '' - else: - break - else: - compiler_path = '' - logger.warning( - 'Found invalid installation for RTools40 on %s', - toolchain_root, - ) - toolchain_root = '' - else: - compiler_path = os.path.join( - toolchain_root, - 'mingw_64' if (sys.maxsize > 2**32) else 'mingw_32', - 'bin', - ) - if os.path.exists(compiler_path): - tool_path = os.path.join(toolchain_root, 'bin') - if not os.path.exists(tool_path): - tool_path = '' - compiler_path = '' - logger.warning( - 'Found invalid installation for RTools35 on %s', - toolchain_root, - ) - toolchain_root = '' - else: - break - else: - compiler_path = '' - logger.warning( - 'Found invalid installation for RTools35 on %s', - toolchain_root, - ) - toolchain_root = '' - else: - toolchain_root = '' - - if not toolchain_root: + for toolchain_root in _rtools_search_roots(install_dir, arch): + found = _probe_rtools_root(toolchain_root, layouts, logger) + if found is not None: + break + + if found is None: raise ValueError( 'no RTools toolchain installation found, ' 'run command line script ' '"python -m cmdstanpy.install_cxx_toolchain"' ) + toolchain_root, compiler_path, tool_path = found + logger.info('Add C++ toolchain to $PATH: %s', toolchain_root) os.environ['PATH'] = ';'.join( list( diff --git a/docsrc/installation.rst b/docsrc/installation.rst index 0f528712..dd74eef6 100644 --- a/docsrc/installation.rst +++ b/docsrc/installation.rst @@ -154,10 +154,13 @@ There is usually a pre-installed C++ compiler as well, but not necessarily new e **MacOS** The Xcode and Xcode command line tools must be installed. Xcode is available for free from the Mac App Store. To install the Xcode command line tools, run the shell command: ``xcode-select --install``. -**Windows** We recommend using the `RTools 4.0 `_ toolchain -which contains a ``g++ 8`` compiler and ``Mingw``, the native Windows equivalent of the GNU-Make utility. +**Windows** We recommend using the `RTools 4.5 `_ toolchain, +which contains a C++ compiler and the GNU-Make build utility. This can be installed along with CmdStan when you invoke the function :meth:`cmdstanpy.install_cmdstan` with argument ``compiler=True``. +RTools is available for both Intel/AMD 64-bit (``x86_64``) and ARM 64-bit (``aarch64``) machines; +CmdStanPy selects the matching build automatically. +Older toolchains (RTools 4.0, Intel/AMD only) are still detected if already installed. .. _install-cmdstan-fun: @@ -218,6 +221,17 @@ machine when running ``install_cmdstan``. If the wrong choice is made, or if you need to manually override this, you can set the ``CMDSTAN_ARCH`` environment variable to one of the above options, or to "false" to use the standard x86 download. +Windows on ARM64 +................ + +CmdStan can be built on ARM 64-bit (``aarch64``) Windows machines using the +ARM64 build of RTools 4.4 or 4.5, which CmdStanPy installs and detects +automatically. + +A Python interpreter built for Intel/AMD and running under emulation reports an +``x86_64`` architecture; in that case the Intel/AMD toolchain is used. The two +architectures can be installed side by side. + DIY Installation ^^^^^^^^^^^^^^^^ diff --git a/test/__init__.py b/test/__init__.py index 69af537c..6ced218f 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -2,8 +2,10 @@ import contextlib import logging +import os import platform import re +import time from importlib import reload from types import ModuleType from typing import Generator, Optional, Type @@ -19,6 +21,28 @@ ) +def delete_file(path: str, timeout: float = 5.0) -> None: + """ + Delete a file, retrying briefly on Windows. + + Antivirus software scans a binary the first time it is executed and + holds it open while doing so, which makes Windows deny the delete with + ``PermissionError`` until the scan finishes. Defender is disabled on the + x86_64 CI images but cannot be disabled on the ARM64 ones. + """ + deadline = time.monotonic() + timeout + while True: + try: + os.remove(path) + return + except FileNotFoundError: + return + except PermissionError: + if platform.system() != 'Windows' or time.monotonic() > deadline: + raise + time.sleep(0.1) + + # pylint: disable=invalid-name @contextlib.contextmanager def raises_nested( diff --git a/test/test_cxx_installation.py b/test/test_cxx_installation.py index bc38c46d..ebe901f7 100644 --- a/test/test_cxx_installation.py +++ b/test/test_cxx_installation.py @@ -1,10 +1,417 @@ """install_cxx_toolchain tests""" +# pylint: disable=redefined-outer-name + +import os +import platform +from pathlib import Path from test import mark_not_windows, mark_windows_only +from typing import Callable import pytest from cmdstanpy import install_cxx_toolchain +from cmdstanpy.utils import cmdstan as cmdstan_utils +from cmdstanpy.utils import cxx_toolchain_path +from cmdstanpy.utils.cmdstan import make_command + +SetArch = Callable[[str], None] + +# (rtools version, arch) -> (compiler subdir, tool subdir, compiler exe) +LAYOUTS = { + ('4.0', 'x86_64'): (('mingw64', 'bin'), ('usr', 'bin'), 'g++'), + ('4.4', 'x86_64'): ( + ('x86_64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + 'g++', + ), + ('4.5', 'x86_64'): ( + ('x86_64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + 'g++', + ), + ('4.4', 'aarch64'): ( + ('aarch64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + 'clang++', + ), + ('4.5', 'aarch64'): ( + ('aarch64-w64-mingw32.static.posix', 'bin'), + ('usr', 'bin'), + 'clang++', + ), +} + +MACHINES = {'x86_64': 'AMD64', 'aarch64': 'ARM64', 'i686': 'x86'} + + +@pytest.fixture(autouse=True) +def clean_env( + monkeypatch: pytest.MonkeyPatch, tmp_path_factory: pytest.TempPathFactory +) -> None: + """Hide any RTools installation the machine already has.""" + for var in ( + 'CMDSTAN_TOOLCHAIN', + 'RTOOLS45_HOME', + 'RTOOLS44_HOME', + 'RTOOLS43_HOME', + 'RTOOLS42_HOME', + 'RTOOLS40_HOME', + ): + monkeypatch.delenv(var, raising=False) + # cxx_toolchain_path mutates PATH in place; let monkeypatch restore it + monkeypatch.setenv('PATH', os.environ.get('PATH', '')) + home = tmp_path_factory.mktemp('home') + monkeypatch.setenv('HOME', str(home)) + monkeypatch.setenv('USERPROFILE', str(home)) + + +@pytest.fixture +def set_arch(monkeypatch: pytest.MonkeyPatch) -> SetArch: + """Pretend to be running on a given architecture.""" + + def _set(arch: str) -> None: + monkeypatch.setattr(platform, 'machine', lambda: MACHINES[arch]) + + return _set + + +def make_toolchain( + root: Path, + version: str, + arch: str, + compiler: bool = True, + tools: bool = True, +) -> str: + """Create a fake RTools installation tree, return its root.""" + compiler_subdir, tool_subdir, compiler_exe = LAYOUTS[(version, arch)] + compiler_dir = root.joinpath(*compiler_subdir) + compiler_dir.mkdir(parents=True, exist_ok=True) + if compiler: + compiler_dir.joinpath(compiler_exe + '.exe').write_text('') + if tools: + root.joinpath(*tool_subdir).mkdir(parents=True, exist_ok=True) + return str(root) + + +# --------------------------------------------------------------------------- +# version / url resolution +# --------------------------------------------------------------------------- + + +@mark_windows_only +@pytest.mark.parametrize( + 'given,expected', + [ + ('4.0', '4.0'), + ('4', '4.0'), + ('40', '4.0'), + ('4.2', '4.2'), + ('42', '4.2'), + ('4.4', '4.4'), + ('44', '4.4'), + ('4.5', '4.5'), + ('45', '4.5'), + # no longer aliased, passed through unchanged + ('3.5', '3.5'), + ('35', '35'), + ], +) +def test_normalize_version(given: str, expected: str) -> None: + assert install_cxx_toolchain.normalize_version(given) == expected + + +@mark_windows_only +def test_toolchain_name() -> None: + assert install_cxx_toolchain.get_toolchain_name() == 'RTools' + + +@mark_windows_only +@pytest.mark.parametrize( + 'version,arch,expected', + [ + ( + '4.5', + 'aarch64', + 'https://github.com/r-hub/rtools45/releases/download/latest/' + 'rtools45-aarch64.exe', + ), + ( + '4.5', + 'x86_64', + 'https://github.com/r-hub/rtools45/releases/download/latest/' + 'rtools45.exe', + ), + ( + '4.4', + 'aarch64', + 'https://github.com/r-hub/rtools44/releases/download/latest/' + 'rtools44-aarch64.exe', + ), + ( + '4.4', + 'x86_64', + 'https://github.com/r-hub/rtools44/releases/download/latest/' + 'rtools44.exe', + ), + ( + '4.0', + 'x86_64', + 'https://cran.r-project.org/bin/windows/Rtools/' + 'rtools40-x86_64.exe', + ), + # no ARM64 builds exist before RTools 4.4 + ('4.0', 'aarch64', ''), + # RTools before 4.0 is no longer supported + ('3.5', 'x86_64', ''), + ('3.5', 'aarch64', ''), + ], +) +def test_get_url(version: str, arch: str, expected: str) -> None: + assert install_cxx_toolchain.get_url(version, arch) == expected + + +@mark_windows_only +@pytest.mark.parametrize('arch', ['x86_64', 'aarch64']) +def test_get_url_defaults_to_current_arch(set_arch: SetArch, arch: str) -> None: + set_arch(arch) + assert install_cxx_toolchain.get_url( + '4.5' + ) == install_cxx_toolchain.get_url('4.5', arch) + + +@mark_windows_only +def test_get_url_unsupported_arch() -> None: + """RTools 4.4 and later dropped 32-bit builds.""" + assert install_cxx_toolchain.get_url('4.5', 'i686') == '' + + +@mark_windows_only +@pytest.mark.parametrize( + 'arch,expected', [('x86_64', '4.5'), ('aarch64', '4.5')] +) +def test_latest_version(set_arch: SetArch, arch: str, expected: str) -> None: + set_arch(arch) + assert install_cxx_toolchain.latest_version() == expected + + +@mark_windows_only +@pytest.mark.parametrize( + 'arch,expected', [('x86_64', 'RTools44'), ('aarch64', 'RTools44-aarch64')] +) +def test_toolchain_folder(set_arch: SetArch, arch: str, expected: str) -> None: + set_arch(arch) + assert ( + install_cxx_toolchain.get_toolchain_version('RTools', '4.4') == expected + ) + + +# --------------------------------------------------------------------------- +# is_installed +# --------------------------------------------------------------------------- + + +@mark_windows_only +@pytest.mark.parametrize('version,arch', sorted(LAYOUTS)) +def test_is_installed( + set_arch: SetArch, tmp_path: Path, version: str, arch: str +) -> None: + set_arch(arch) + root = make_toolchain(tmp_path, version, arch) + assert install_cxx_toolchain.is_installed(root, version) + + +@mark_windows_only +@pytest.mark.parametrize('version,arch', sorted(LAYOUTS)) +def test_is_installed_incomplete( + set_arch: SetArch, tmp_path: Path, version: str, arch: str +) -> None: + """Compiler directory exists but the compiler itself is missing.""" + set_arch(arch) + root = make_toolchain(tmp_path, version, arch, compiler=False) + assert not install_cxx_toolchain.is_installed(root, version) + + +@mark_windows_only +def test_is_installed_missing_compiler( + set_arch: SetArch, tmp_path: Path +) -> None: + """Both directories present, but no compiler binary in them.""" + set_arch('aarch64') + make_toolchain(tmp_path, '4.4', 'aarch64', compiler=False) + assert not install_cxx_toolchain.is_installed(str(tmp_path), '4.4') + + +@mark_windows_only +def test_is_installed_wrong_arch(set_arch: SetArch, tmp_path: Path) -> None: + """An ARM64 install must not be reported to an x86_64 process.""" + set_arch('aarch64') + root = make_toolchain(tmp_path, '4.4', 'aarch64') + set_arch('x86_64') + assert not install_cxx_toolchain.is_installed(root, '4.4') + + +@mark_windows_only +def test_is_installed_unknown_version( + set_arch: SetArch, tmp_path: Path +) -> None: + set_arch('x86_64') + root = make_toolchain(tmp_path, '4.4', 'x86_64') + assert not install_cxx_toolchain.is_installed(root, '9.9') + + +# --------------------------------------------------------------------------- +# cxx_toolchain_path +# --------------------------------------------------------------------------- + + +@mark_windows_only +@pytest.mark.parametrize('version,arch', sorted(LAYOUTS)) +def test_toolchain_path_from_install_dir( + set_arch: SetArch, tmp_path: Path, version: str, arch: str +) -> None: + set_arch(arch) + folder = install_cxx_toolchain.get_toolchain_version('RTools', version) + make_toolchain(tmp_path / folder, version, arch) + + compiler_path, tool_path = cxx_toolchain_path(version, str(tmp_path)) + + compiler_subdir, tool_subdir, _ = LAYOUTS[(version, arch)] + assert compiler_path == str(tmp_path.joinpath(folder, *compiler_subdir)) + assert tool_path == str(tmp_path.joinpath(folder, *tool_subdir)) + assert os.environ['PATH'].startswith(f'{compiler_path};{tool_path};') + + +@mark_windows_only +@pytest.mark.parametrize('version,arch', sorted(LAYOUTS)) +def test_toolchain_path_from_env( + monkeypatch: pytest.MonkeyPatch, + set_arch: SetArch, + tmp_path: Path, + version: str, + arch: str, +) -> None: + set_arch(arch) + root = make_toolchain(tmp_path, version, arch) + monkeypatch.setenv('CMDSTAN_TOOLCHAIN', root) + + compiler_subdir, tool_subdir, _ = LAYOUTS[(version, arch)] + assert cxx_toolchain_path() == ( + str(tmp_path.joinpath(*compiler_subdir)), + str(tmp_path.joinpath(*tool_subdir)), + ) + + +@mark_windows_only +@pytest.mark.parametrize('arch', ['x86_64', 'aarch64']) +def test_toolchain_path_unversioned_prefers_newest( + set_arch: SetArch, tmp_path: Path, arch: str +) -> None: + """With no version requested, 4.5 wins over 4.4.""" + set_arch(arch) + old = install_cxx_toolchain.get_toolchain_version('RTools', '4.4') + new = install_cxx_toolchain.get_toolchain_version('RTools', '4.5') + make_toolchain(tmp_path / old, '4.4', arch) + make_toolchain(tmp_path / new, '4.5', arch) + + compiler_path, _ = cxx_toolchain_path(None, str(tmp_path)) + assert new in compiler_path + + +@mark_windows_only +def test_toolchain_path_rtools_home( + monkeypatch: pytest.MonkeyPatch, set_arch: SetArch, tmp_path: Path +) -> None: + set_arch('x86_64') + root = make_toolchain(tmp_path, '4.4', 'x86_64') + monkeypatch.setenv('RTOOLS44_HOME', root) + compiler_path, _ = cxx_toolchain_path() + assert compiler_path.startswith(root) + + +@mark_windows_only +def test_toolchain_path_incomplete_warns( + monkeypatch: pytest.MonkeyPatch, + set_arch: SetArch, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + set_arch('x86_64') + root = make_toolchain(tmp_path, '4.4', 'x86_64', tools=False) + monkeypatch.setenv('CMDSTAN_TOOLCHAIN', root) + with pytest.raises(ValueError, match='no RTools toolchain installation'): + cxx_toolchain_path() + assert 'Found invalid RTools installation' in caplog.text + + +@mark_windows_only +def test_toolchain_path_ignores_empty_stub_dirs( + monkeypatch: pytest.MonkeyPatch, set_arch: SetArch, tmp_path: Path +) -> None: + """RTools 4.2+ ship empty mingw64/ucrt64/clangarm64 directories.""" + set_arch('x86_64') + make_toolchain(tmp_path, '4.5', 'x86_64') + for stub in ('mingw64', 'mingw32', 'ucrt64', 'clang64', 'clangarm64'): + tmp_path.joinpath(stub, 'bin').mkdir(parents=True) + monkeypatch.setenv('CMDSTAN_TOOLCHAIN', str(tmp_path)) + + # an explicit 4.0 request must not match the empty mingw64 stub + with pytest.raises(ValueError, match='no RTools toolchain installation'): + cxx_toolchain_path('4.0') + + compiler_path, _ = cxx_toolchain_path() + assert compiler_path.endswith( + os.path.join('x86_64-w64-mingw32.static.posix', 'bin') + ) + + +@mark_windows_only +def test_toolchain_path_unrecognized_root( + monkeypatch: pytest.MonkeyPatch, + set_arch: SetArch, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """A directory that exists but matches no known layout.""" + set_arch('x86_64') + monkeypatch.setenv('CMDSTAN_TOOLCHAIN', str(tmp_path)) + with pytest.raises(ValueError, match='no RTools toolchain installation'): + cxx_toolchain_path() + assert 'Found no usable RTools installation' in caplog.text + + +@mark_windows_only +def test_toolchain_path_not_found( + monkeypatch: pytest.MonkeyPatch, set_arch: SetArch, tmp_path: Path +) -> None: + set_arch('x86_64') + monkeypatch.setenv('CMDSTAN_TOOLCHAIN', str(tmp_path / 'nowhere')) + with pytest.raises(ValueError, match='no RTools toolchain installation'): + cxx_toolchain_path() + + +@mark_windows_only +def test_toolchain_path_unsupported_version(set_arch: SetArch) -> None: + set_arch('aarch64') + with pytest.raises(ValueError, match='unsupported RTools version'): + cxx_toolchain_path('4.0') + + +@mark_windows_only +def test_toolchain_path_bad_version_type() -> None: + with pytest.raises(TypeError, match='Format version number as a string'): + cxx_toolchain_path(4.0) # type: ignore[arg-type] + + +@mark_not_windows +def test_cxx_toolchain_path_not_windows() -> None: + with pytest.raises(RuntimeError, match='only supported on Windows'): + cxx_toolchain_path() + + +# --------------------------------------------------------------------------- +# misc +# --------------------------------------------------------------------------- @mark_windows_only @@ -40,17 +447,69 @@ def test_install_not_windows() -> None: @mark_windows_only -def test_normalize_version() -> None: - """Test supported versions.""" +def test_install_unsupported_combination( + set_arch: SetArch, tmp_path: Path +) -> None: + """RTools 4.0 has no ARM64 build.""" + set_arch('aarch64') + with pytest.raises(ValueError, match='not available for aarch64'): + install_cxx_toolchain.run_rtools_install( + {'version': '4.0', 'dir': str(tmp_path)} + ) - for ver in ['4.0', '4', '40']: - assert install_cxx_toolchain.normalize_version(ver) == '4.0' - for ver in ['3.5', '35']: - assert install_cxx_toolchain.normalize_version(ver) == '3.5' +@mark_windows_only +def test_install_defaults_to_latest_version( + monkeypatch: pytest.MonkeyPatch, set_arch: SetArch, tmp_path: Path +) -> None: + set_arch('x86_64') + monkeypatch.setattr(install_cxx_toolchain, 'latest_version', lambda: '9.9') + with pytest.raises(ValueError, match='RTools 9.9 is not available'): + install_cxx_toolchain.run_rtools_install( + {'version': None, 'dir': str(tmp_path)} + ) + + +# --------------------------------------------------------------------------- +# make resolution +# --------------------------------------------------------------------------- + + +def test_make_command_honours_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv('MAKE', 'my-make') + assert make_command() == 'my-make' + + +@mark_not_windows +def test_make_command_posix(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv('MAKE', raising=False) + assert make_command() == 'make' @mark_windows_only -def test_toolchain_name() -> None: - """Check toolchain name.""" - assert install_cxx_toolchain.get_toolchain_name() == 'RTools' +@pytest.mark.parametrize( + 'available,expected', + [ + # RTools 4.0 installs mingw32-make, which stays preferred + (['mingw32-make', 'make'], 'mingw32-make'), + # RTools 4.2+ only ship plain make in usr/bin + (['make'], 'make'), + ([], 'make'), + ], +) +def test_make_command_windows( + monkeypatch: pytest.MonkeyPatch, available: list[str], expected: str +) -> None: + monkeypatch.delenv('MAKE', raising=False) + monkeypatch.setattr( + cmdstan_utils.shutil, + 'which', + lambda name: f'C:\\fake\\{name}.exe' if name in available else None, + ) + assert make_command() == expected + + +@mark_windows_only +def test_usage(capsys: pytest.CaptureFixture) -> None: + install_cxx_toolchain.usage() + assert '--version' in capsys.readouterr().out diff --git a/test/test_model.py b/test/test_model.py index 8334560a..f2292b14 100644 --- a/test/test_model.py +++ b/test/test_model.py @@ -5,7 +5,7 @@ import re import shutil import tempfile -from test import check_present +from test import check_present, delete_file from unittest.mock import patch import numpy as np @@ -43,12 +43,6 @@ def test_model_good() -> None: assert os.path.samefile(model.exe_file, BERN_EXE) assert 'bernoulli' == model.name - # compile with external header - model = CmdStanModel( - stan_file=os.path.join(DATAFILES_PATH, "external.stan"), - user_header=os.path.join(DATAFILES_PATH, 'return_one.hpp'), - ) - # default model name model = CmdStanModel(stan_file=BERN_STAN) assert BERN_BASENAME == model.name @@ -59,6 +53,16 @@ def test_model_good() -> None: assert os.path.samefile(model.exe_file, BERN_EXE) +def test_model_compile_user_header() -> None: + """The header path is passed with this platform's native separators.""" + model = CmdStanModel( + stan_file=os.path.join(DATAFILES_PATH, "external.stan"), + user_header=os.path.join(DATAFILES_PATH, 'return_one.hpp'), + force_compile=True, + ) + assert os.path.exists(model.exe_file) + + def test_ctor_compile_arg() -> None: if os.path.exists(BERN_EXE): os.remove(BERN_EXE) @@ -154,7 +158,7 @@ def test_model_info() -> None: info_dict = model.exe_info() assert info_dict['STAN_THREADS'].lower() == 'false' - os.remove(model.exe_file) + delete_file(model.exe_file) with pytest.raises(RuntimeError): model.exe_info() @@ -299,7 +303,7 @@ def test_model_paths() -> None: assert model1.stan_file == dotdot_stan assert model1.exe_file == dotdot_exe os.remove(dotdot_stan) - os.remove(dotdot_exe) + delete_file(dotdot_exe) tilde_stan = os.path.realpath( os.path.join(os.path.expanduser('~'), 'bernoulli.stan') @@ -316,7 +320,7 @@ def test_model_paths() -> None: assert model2.stan_file == tilde_stan assert model2.exe_file == tilde_exe os.remove(tilde_stan) - os.remove(tilde_exe) + delete_file(tilde_exe) def test_model_none() -> None: @@ -362,7 +366,7 @@ def test_model_compile() -> None: @pytest.mark.parametrize("path", ["space in path", "tilde~in~path"]) def test_model_compile_special_char(path: str) -> None: with tempfile.TemporaryDirectory( - prefix="cmdstanpy_testfolder_" + prefix="cmdstanpy_testfolder_", ignore_cleanup_errors=True ) as tmp_path: path_with_special_char = os.path.join(tmp_path, path) os.makedirs(path_with_special_char, exist_ok=True)