diff --git a/.gitmodules b/.gitmodules index 7b8f5ca..546361d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "OpenBLAS"] path = OpenBLAS url = https://github.com/xianyi/OpenBLAS.git -[submodule "gfortran-install"] - path = gfortran-install - url = https://github.com/MacPython/gfortran-install.git diff --git a/gfortran-install b/gfortran-install deleted file mode 160000 index 3dd38d9..0000000 --- a/gfortran-install +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3dd38d9ce78b3890598cb0eff18a7bec50c06f5e diff --git a/pyproject.toml b/pyproject.toml index c956d4c..eb3981c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "scipy-openblas64" # v0.3.30-349-gf6df9beb -version = "0.3.30.349.0" +version = "0.3.30.349.1" requires-python = ">=3.7" description = "Provides OpenBLAS for python packaging" readme = "README.md" diff --git a/tools/build_steps.sh b/tools/build_steps.sh index 6aacb01..de1017c 100644 --- a/tools/build_steps.sh +++ b/tools/build_steps.sh @@ -24,25 +24,20 @@ function get_os { function before_build { - # Manylinux Python version set in build_lib + if [ ! -e /usr/local/lib ]; then + sudo mkdir -p /usr/local/lib + sudo chmod 777 /usr/local/lib + touch /usr/local/lib/.dir_exists + fi + if [ ! -e /usr/local/include ]; then + sudo mkdir -p /usr/local/include + sudo chmod 777 /usr/local/include + touch /usr/local/include/.dir_exists + fi if [ -n "$IS_OSX" ]; then - if [ ! -e /usr/local/lib ]; then - sudo mkdir -p /usr/local/lib - sudo chmod 777 /usr/local/lib - touch /usr/local/lib/.dir_exists - fi - if [ ! -e /usr/local/include ]; then - sudo mkdir -p /usr/local/include - sudo chmod 777 /usr/local/include - touch /usr/local/include/.dir_exists - fi # get_macpython_environment ${MB_PYTHON_VERSION} venv python3.9 -m venv venv source venv/bin/activate - alias gfortran=gfortran-15 - # Deployment target set by gfortran_utils - echo "Deployment target $MACOSX_DEPLOYMENT_TARGET" - # Build the objconv tool (cd ${ROOT_DIR}/objconv && bash ../tools/build_objconv.sh) fi @@ -55,13 +50,9 @@ function clean_code { [ -z "$build_commit" ] && echo "build_commit not defined" && exit 1 pushd OpenBLAS git fetch origin --tags - echo after git fetch origin git checkout $build_commit - echo after git checkout $build_commit git clean -fxd - echo after git clean git submodule update --init --recursive - echo after git submodule update popd } @@ -110,9 +101,9 @@ function build_lib { local manylinux=${MB_ML_VER:-1} if [ -n "$IS_OSX" ]; then # Do build, add gfortran hash to end of name - do_build_lib "$plat" "gf_${GFORTRAN_SHA:0:7}" "$interface64" "$nightly" + do_build_lib "$plat" "$interface64" "$nightly" else - do_build_lib "$plat" "" "$interface64" "$nightly" + do_build_lib "$plat" "$interface64" "$nightly" fi } @@ -129,8 +120,6 @@ function do_build_lib { # Build openblas lib # Input arg # plat - one of i686, x86_64, arm64 - # suffix (optional) - suffix for output archive name - # Suffix added with hyphen prefix # interface64 (optional) - whether to build ILP64 openblas # with 64_ symbol suffix # nightly (optional) - whether to build for nightlies @@ -138,9 +127,8 @@ function do_build_lib { # Depends on globals # BUILD_PREFIX - install suffix e.g. "/usr/local" local plat=$1 - local suffix=$2 - local interface64=$3 - local nightly=$4 + local interface64=$2 + local nightly=$3 case $(get_os)-$plat in Linux-x86_64) local bitness=64 @@ -150,18 +138,25 @@ function do_build_lib { Darwin-x86_64) local bitness=64 local target="CORE2" - # Use gfortran-11 - unalias gfortran + # Use gfortran from conda # Since install_fortran uses `uname -a` to determine arch, - # force the architecture - arch -${PLAT} bash -s << EOF -source ${ROOT_DIR}/gfortran-install/gfortran_utils.sh -install_gfortran -EOF + # force the architecture when using rosetta + unalias gfortran || true + arch -${PLAT} bash -s << " EOF" + set -xe + source tools/gfortran_utils.sh + install_gfortran + EOF + # re-export these, since we ran in a shell + export FC=$(find /opt/gfortran/gfortran-darwin-${PLAT}-native/bin -name "*-gfortran") + local libdir=/opt/gfortran/gfortran-darwin-${PLAT}-native/lib + export FFLAGS="-L${libdir} -Wl,-rpath,${libdir}" + export DYLD_LIBRARY_PATH=/usr/local/lib:$DYLD_LIBRARY_PATH CFLAGS="$CFLAGS -arch x86_64" export SDKROOT=${SDKROOT:-$(xcrun --show-sdk-path)} local dynamic_list="CORE2 NEHALEM SANDYBRIDGE HASWELL SKYLAKEX" + MACOSX_DEPLOYMENT_TARGET="10.9" ;; *-i686) local bitness=32 @@ -178,6 +173,18 @@ EOF fi ;; Darwin-arm64) + export FC=gfortran-15 + which $FC + # guess? + local libdir=/opt/homebrew/Cellar/gcc/15.2.0/lib/gcc/current + if [ ! -e $libdir/libgfortran.a ]; then + echo where is libfortran.a? + find /opt -name libgfortran.a + exit 1 + fi + export FFLAGS="-L${libdir} -Wl,-rpath,${libdir}" + export GFORTRAN_LIBDIR=$($FC -print-file-name=libgfortran.dylib | xargs dirname) + export LDFLAGS="-L/opt/homebrew/opt/llvm/lib -L$GFORTRAN_LIBDIR" local bitness=64 local target="VORTEX" CFLAGS="$CFLAGS -ftrapping-math -mmacos-version-min=11.0" @@ -187,6 +194,7 @@ EOF ;; *-s390x) local bitness=64 + local target="ZARCH_GENERIC" ;; *-ppc64le) local bitness=64 @@ -216,7 +224,10 @@ EOF git config --global --add safe.directory '*' pushd OpenBLAS patch_source + # force failure when fortran compiler is no detected + sed -i.bak -e 's/info OpenBLAS: Detecting fortran compiler failed/error OpenBLAS: Detecting fortran compiler failed/g' Makefile echo start building + if [ "$plat" == "loongarch64" ]; then # https://github.com/OpenMathLib/OpenBLAS/blob/develop/.github/workflows/loongarch64.yml#L65 echo -n > utest/test_dsdot.c @@ -225,14 +236,14 @@ EOF fi if [ -n "$dynamic_list" ]; then CFLAGS="$CFLAGS -fvisibility=protected -Wno-uninitialized" \ - make BUFFERSIZE=20 DYNAMIC_ARCH=1 QUIET_MAKE=1 \ + make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ USE_OPENMP=0 NUM_THREADS=64 \ DYNAMIC_LIST="$dynamic_list" \ BINARY="$bitness" $interface_flags \ TARGET="$target" else CFLAGS="$CFLAGS -fvisibility=protected -Wno-uninitialized" \ - make BUFFERSIZE=20 DYNAMIC_ARCH=1 QUIET_MAKE=1 \ + make BUFFERSIZE=20 DYNAMIC_ARCH=1 \ USE_OPENMP=0 NUM_THREADS=64 \ BINARY="$bitness" $interface_flags \ TARGET="$target" @@ -246,8 +257,6 @@ EOF fi mv $BUILD_PREFIX/lib/pkgconfig/openblas*.pc $BUILD_PREFIX/lib/pkgconfig/scipy-openblas.pc local plat_tag=$(get_plat_tag $plat) - local suff="" - [ -n "$suffix" ] && suff="-$suffix" if [ "$interface64" = "1" ]; then # OpenBLAS does not install the symbol suffixed static library, # do it ourselves @@ -260,7 +269,7 @@ EOF rm $BUILD_PREFIX/lib/pkgconfig/scipy-openblas.pc.bak fi - local out_name="openblas${symbolsuffix}-${version}-${plat_tag}${suff}.tar.gz" + local out_name="openblas${symbolsuffix}-${version}-${plat_tag}.tar.gz" tar zcvf libs/$out_name \ $BUILD_PREFIX/include/*blas* \ $BUILD_PREFIX/include/*lapack* \ diff --git a/tools/gfortran_utils.sh b/tools/gfortran_utils.sh index b7127f5..74ac108 100644 --- a/tools/gfortran_utils.sh +++ b/tools/gfortran_utils.sh @@ -16,168 +16,76 @@ # Bash utilities for use with gfortran -ARCHIVE_SDIR="${ARCHIVE_SDIR:-archives}" - -GF_UTIL_DIR=$(dirname "${BASH_SOURCE[0]}") - -function get_distutils_platform { - # Report platform as in form of distutils get_platform. - # This is like the platform tag that pip will use. - # Modify fat architecture tags on macOS to reflect compiled architecture - - # Deprecate this function once get_distutils_platform_ex is used in all - # downstream projects - local plat=$1 - case $plat in - i686|x86_64|arm64|universal2|intel|aarch64|s390x|ppc64le) ;; - *) echo Did not recognize plat $plat; return 1 ;; - esac - local uname=${2:-$(uname)} - if [ "$uname" != "Darwin" ]; then - if [ "$plat" == "intel" ]; then - echo plat=intel not allowed for Manylinux - return 1 - fi - echo "manylinux1_$plat" - return - fi - # macOS 32-bit arch is i386 - [ "$plat" == "i686" ] && plat="i386" - local target=$(echo $MACOSX_DEPLOYMENT_TARGET | tr .- _) - echo "macosx_${target}_${plat}" -} - -function get_distutils_platform_ex { - # Report platform as in form of distutils get_platform. - # This is like the platform tag that pip will use. - # Modify fat architecture tags on macOS to reflect compiled architecture - # For non-darwin, report manylinux version - local plat=$1 - local mb_ml_ver=${MB_ML_VER:-1} - case $plat in - i686|x86_64|arm64|universal2|intel|aarch64|s390x|ppc64le) ;; - *) echo Did not recognize plat $plat; return 1 ;; - esac - local uname=${2:-$(uname)} - if [ "$uname" != "Darwin" ]; then - if [ "$plat" == "intel" ]; then - echo plat=intel not allowed for Manylinux - return 1 - fi - echo "manylinux${mb_ml_ver}_${plat}" - return - fi - # macOS 32-bit arch is i386 - [ "$plat" == "i686" ] && plat="i386" - local target=$(echo $MACOSX_DEPLOYMENT_TARGET | tr .- _) - echo "macosx_${target}_${plat}" -} - -function get_macosx_target { - # Report MACOSX_DEPLOYMENT_TARGET as given by distutils get_platform. - python3 -c "import sysconfig as s; print(s.get_config_vars()['MACOSX_DEPLOYMENT_TARGET'])" -} - function check_gfortran { # Check that gfortran exists on the path - if [ -z "$(which gfortran)" ]; then + if [[ -n "$FC" && -e "$FC" ]]; then + echo using gfortran from FC + echo $FC --version + elif [ -z "$(which gfortran)" ]; then echo Missing gfortran exit 1 + else + echo using gfortran on the PATH + echo gfortran --version fi } -function get_gf_lib_for_suf { - local suffix=$1 - local prefix=$2 - local plat=${3:-$PLAT} - local uname=${4:-$(uname)} - if [ -z "$prefix" ]; then echo Prefix not defined; exit 1; fi - local plat_tag=$(get_distutils_platform_ex $plat $uname) - if [ -n "$suffix" ]; then suffix="-$suffix"; fi - local fname="$prefix-${plat_tag}${suffix}.tar.gz" - local out_fname="${ARCHIVE_SDIR}/$fname" - [ -s $out_fname ] || (echo "$out_fname is empty"; exit 24) - echo "$out_fname" -} - if [ "$(uname)" == "Darwin" ]; then - mac_target=${MACOSX_DEPLOYMENT_TARGET:-$(get_macosx_target)} - export MACOSX_DEPLOYMENT_TARGET=$mac_target - # Keep this for now as some builds might depend on this being - # available before install_gfortran is called - export GFORTRAN_SHA=c469a420d2d003112749dcdcbe3c684eef42127e - # Set SDKROOT env variable if not set - export SDKROOT=${SDKROOT:-$(xcrun --show-sdk-path)} function download_and_unpack_gfortran { - local arch=$1 - local type=$2 - curl -L -O https://github.com/isuruf/gcc/releases/download/gcc-11.3.0-2/gfortran-darwin-${arch}-${type}.tar.gz - case ${arch}-${type} in - arm64-native) - export GFORTRAN_SHA=0d5c118e5966d0fb9e7ddb49321f63cac1397ce8 - ;; - arm64-cross) - export GFORTRAN_SHA=527232845abc5af21f21ceacc46fb19c190fe804 - ;; - x86_64-native) - export GFORTRAN_SHA=c469a420d2d003112749dcdcbe3c684eef42127e - ;; - x86_64-cross) - export GFORTRAN_SHA=107604e57db97a0ae3e7ca7f5dd722959752f0b3 - ;; - esac - if [[ "$(shasum gfortran-darwin-${arch}-${type}.tar.gz)" != "${GFORTRAN_SHA} gfortran-darwin-${arch}-${type}.tar.gz" ]]; then - echo "shasum mismatch for gfortran-darwin-${arch}-${type}" + local arch=$1 + local type=$2 + local gccver=gcc-15.2.0 + case ${arch}-${type} in + arm64-native) + export GFORTRAN_SHA=999a91eef894d32f99e3b641520bef9f475055067f301f0f1947b8b716b5922a + ;; + arm64-cross) + export GFORTRAN_SHA=39ef2590629c2f238f1a67469fa429d8d6362425b277abb57fd2f3c982568a3f + ;; + x86_64-native) + #override gccver + gccver=gcc-11.3.0-2 + export GFORTRAN_SHA=981367dd0ad4335613e91bbee453d60b6669f5d7e976d18c7bdb7f1966f26ae4 + ;; + x86_64-cross) + export GFORTRAN_SHA=0a19ca91019a75501e504eed1cad2be6ea92ba457ec815beb0dd28652eb0ce3f + ;; + *) echo Did not recognize arch-plat $arch-$plat; return 1 ;; + esac + curl -L -O https://github.com/isuruf/gcc/releases/download/${gccver}/gfortran-darwin-${arch}-${type}.tar.gz + ls -lh gfortran-darwin-${arch}-${type}.tar.gz + local filesha=$(python3 tools/sha256sum.py gfortran-darwin-${arch}-${type}.tar.gz) + if [[ "$filesha" != "${GFORTRAN_SHA}" ]]; then + echo shasum mismatch for ${gccver}/gfortran-darwin-${arch}-${type} + echo expected $GFORTRAN_SHA + echo got $filesha exit 1 fi - sudo mkdir -p /opt/ - sudo cp "gfortran-darwin-${arch}-${type}.tar.gz" /opt/gfortran-darwin-${arch}-${type}.tar.gz - pushd /opt - sudo tar -xvf gfortran-darwin-${arch}-${type}.tar.gz - sudo rm gfortran-darwin-${arch}-${type}.tar.gz + if [[ ! -e /opt/gfortran ]]; then + sudo mkdir -p /opt/gfortran + sudo chmod 777 /opt/gfortran + fi + cp "gfortran-darwin-${arch}-${type}.tar.gz" /opt/gfortran/gfortran-darwin-${arch}-${type}.tar.gz + pushd /opt/gfortran + tar -xvf gfortran-darwin-${arch}-${type}.tar.gz + rm gfortran-darwin-${arch}-${type}.tar.gz popd - if [[ "${type}" == "native" ]]; then - # Link these into /usr/local so that there's no need to add rpath or -L - for f in libgfortran.dylib libgfortran.5.dylib libgcc_s.1.dylib libgcc_s.1.1.dylib libquadmath.dylib libquadmath.0.dylib; do - ln -sf /opt/gfortran-darwin-${arch}-${type}/lib/$f /usr/local/lib/$f - done - # Add it to PATH - ln -sf /opt/gfortran-darwin-${arch}-${type}/bin/gfortran /usr/local/bin/gfortran - fi - } - - function install_arm64_cross_gfortran { - download_and_unpack_gfortran arm64 cross - export FC_ARM64="$(find /opt/gfortran-darwin-arm64-cross/bin -name "*-gfortran")" - local libgfortran="$(find /opt/gfortran-darwin-arm64-cross/lib -name libgfortran.dylib)" + export FC="$(find /opt/gfortran/gfortran-darwin-${arch}-${type}/bin -name "*-gfortran")" + local libgfortran="$(find /opt/gfortran/gfortran-darwin-${arch}-${type}/lib -name libgfortran.dylib)" local libdir=$(dirname $libgfortran) - - export FC_ARM64_LDFLAGS="-L$libdir -Wl,-rpath,$libdir" - if [[ "${PLAT:-}" == "arm64" ]]; then - export FC=$FC_ARM64 - fi + export FFLAGS="-L$libdir -Wl,-rpath,$libdir" } + function install_gfortran { download_and_unpack_gfortran $(uname -m) native check_gfortran - if [[ "${PLAT:-}" == "universal2" || "${PLAT:-}" == "arm64" ]]; then - install_arm64_cross_gfortran - fi } - function get_gf_lib { - # Get lib with gfortran suffix - get_gf_lib_for_suf "gf_${GFORTRAN_SHA:0:7}" $@ - } else function install_gfortran { # No-op - already installed on manylinux image check_gfortran } - function get_gf_lib { - # Get library with no suffix - get_gf_lib_for_suf "" $@ - } fi diff --git a/tools/local_build.sh b/tools/local_build.sh index 4754c2c..a436b26 100755 --- a/tools/local_build.sh +++ b/tools/local_build.sh @@ -5,8 +5,12 @@ set -e # Set extra env if [[ $(uname) == "Darwin" ]]; then - # Force x86_64 - export PLAT=x86_64 + # export PLAT=x86_64 + export PLAT=arm64 + # Force installation of gfortran + export IS_MACOS=1 + export IS_OSX=1 + elif [[ $(uname -m) == "x86_64" ]]; then echo got x86_64 export PLAT=x86_64 diff --git a/tools/sha256sum.py b/tools/sha256sum.py new file mode 100755 index 0000000..da39d6e --- /dev/null +++ b/tools/sha256sum.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +import hashlib +import sys + +filename = sys.argv[1] +sha256_hash = hashlib.sha256() +with open(filename,"rb") as f: + # Read and update hash string value in blocks + for byte_block in iter(lambda: f.read(32768),b""): + sha256_hash.update(byte_block) + print(sha256_hash.hexdigest()) +