From 13d9eee23fc455613c458c38217c8c86323b5b6a Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 25 Nov 2021 20:33:10 +0000 Subject: [PATCH 1/8] Download from conda --- hdf5-conda/Cargo.toml | 16 ++++++ hdf5-conda/build.rs | 110 ++++++++++++++++++++++++++++++++++++++++++ hdf5-conda/src/lib.rs | 1 + hdf5-sys/Cargo.toml | 2 + hdf5-sys/build.rs | 18 +++++++ 5 files changed, 147 insertions(+) create mode 100644 hdf5-conda/Cargo.toml create mode 100644 hdf5-conda/build.rs create mode 100644 hdf5-conda/src/lib.rs diff --git a/hdf5-conda/Cargo.toml b/hdf5-conda/Cargo.toml new file mode 100644 index 000000000..0dc73cb78 --- /dev/null +++ b/hdf5-conda/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "hdf5-conda" +version = "0.0.1" # !V +keywords = ["hdf5"] +build = "build.rs" +repository = "https://github.com/aldanor/hdf5-rust" +homepage = "https://github.com/aldanor/hdf5-rust" +description = "Build script for including hdf5 from conda." +edition = "2018" +links = "hdf5conda" + +[build-dependencies] +attohttpc = { version = "0.18.0", default-features = false, features = ["compress", "tls-rustls"] } +bzip2 = "0.4.3" +cfg-if = "1.0.0" +tar = "0.4.37" diff --git a/hdf5-conda/build.rs b/hdf5-conda/build.rs new file mode 100644 index 000000000..638049ff4 --- /dev/null +++ b/hdf5-conda/build.rs @@ -0,0 +1,110 @@ +use std::env; +use std::fs::File; +use std::io; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use bzip2::read::BzDecoder; +use cfg_if::cfg_if; +use tar::Archive; + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + + let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + + let uri = format!("{}/{}", DOWNLOAD_URL, DOWNLOAD_BINARY); + let archive_path = out_dir.join("hdf5-conda.tar.bz2"); + if archive_path.exists() { + println!("Using existing archive"); + } else { + println!("Download archive"); + download(&uri, "hdf5-conda.tar.bz2", &out_dir); + extract(&archive_path, &out_dir); + } + + let inc_dir = out_dir.join(INC_PATH); + let root_dir = out_dir.join(LIB_PATH); + + println!("cargo:library={}", LIB_NAME); + println!("cargo:root={}", root_dir.display()); + println!("cargo:include={}", inc_dir.display()); +} + +pub const DOWNLOAD_URL: &str = "https://anaconda.org/conda-forge/hdf5/1.12.1/download"; + +const INC_PATH: &str = { + cfg_if! { + if #[cfg(any(target_os = "linux", target_os = "macos"))] { + "include" + } else if #[cfg(target_os = "windows")] { + "Library\\include" + } else { + compile_error!("This crate can not be used on this platform"); + } + } +}; + +const LIB_NAME: &str = { + cfg_if! { + if #[cfg(any(target_os = "linux", target_os = "macos"))] { + "hdf5" + } else if #[cfg(target_os = "windows")] { + "libhdf5" + } else { + compile_error!("This crate can not be used on this platform"); + } + } +}; + +const LIB_PATH: &str = { + cfg_if! { + if #[cfg(any(target_os = "linux", target_os = "macos"))] { + "lib" + } else if #[cfg(target_os = "windows")] { + "Library\\lib" + } else { + compile_error!("This crate can not be used on this platform"); + } + } +}; + +const DOWNLOAD_BINARY: &str = { + cfg_if! { + if #[cfg(all(target_os = "linux", target_arch = "x86_64"))] { + "linux-64/hdf5-1.12.1-nompi_h2750804_102.tar.bz2" + } else if #[cfg(all(target_os = "linux", target_arch = "aarch64"))] { + "linux-aarch64/hdf5-1.12.1-nompi_h774d4d8_102.tar.bz2" + } else if #[cfg(any(target_os = "osx", target_arch = "x86_64"))] { + "osx-64/hdf5-1.12.1-nompi_hd9e8a45_102.tar.bz2" + } else if #[cfg(any(target_os = "windows", target_arch = "x86_64"))] { + "hdf5-1.12.1-nompi_h2a0e4a3_102.tar.bz2" + } else { + compile_error!("This package can not be used on this arch"); + } + } +}; + +fn download(uri: &str, filename: &str, out_dir: &Path) { + let out = PathBuf::from(out_dir.join(filename)); + + let f = File::create(&out).unwrap(); + let writer = io::BufWriter::new(f); + + let req = attohttpc::get(uri).read_timeout(Duration::new(90, 0)); + + let response = req.send().unwrap(); + + if !response.is_success() { + panic!("Unexpected response code {:?} for {}", response.status(), uri); + } + + response.write_to(writer).unwrap(); +} + +fn extract, P2: AsRef>(archive_path: P, extract_to: P2) { + let file = File::open(archive_path).unwrap(); + let unzipped = BzDecoder::new(file); + let mut a = Archive::new(unzipped); + a.unpack(extract_to).unwrap(); +} diff --git a/hdf5-conda/src/lib.rs b/hdf5-conda/src/lib.rs new file mode 100644 index 000000000..412d63058 --- /dev/null +++ b/hdf5-conda/src/lib.rs @@ -0,0 +1 @@ +//! Dummy crate for linking to hdf5 from conda diff --git a/hdf5-sys/Cargo.toml b/hdf5-sys/Cargo.toml index a318eb70a..15d7de5a7 100644 --- a/hdf5-sys/Cargo.toml +++ b/hdf5-sys/Cargo.toml @@ -18,6 +18,7 @@ libc = "0.2" mpi-sys = { version = "0.1", optional = true } libz-sys = { version = "1.0.25", optional = true, default-features = false } hdf5-src = { path = "../hdf5-src", version = "0.8.1", optional = true } # !V +hdf5-conda = { path = "../hdf5-conda", version = "0.0.1", optional = true } # !V # Please see README for further explanation of these feature flags [features] @@ -28,6 +29,7 @@ threadsafe = ["hdf5-src/threadsafe"] zlib = ["libz-sys", "hdf5-src/zlib"] static = ["hdf5-src"] deprecated = ["hdf5-src/deprecated"] +conda = ["hdf5-conda"] [build-dependencies] libloading = "0.7" diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index 6da3daa80..7bb603a6b 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -639,6 +639,8 @@ impl Config { fn main() { if feature_enabled("STATIC") && std::env::var_os("HDF5_DIR").is_none() { get_build_and_emit(); + } else if feature_enabled("CONDA") && std::env::var_os("HDF5_DIR").is_none() { + link_to_conda(); } else { let mut searcher = LibrarySearcher::new_from_env(); searcher.try_locate_hdf5_library(); @@ -680,3 +682,19 @@ fn get_build_and_emit() { let config = Config { header, inc_dir: "".into(), link_paths: Vec::new() }; config.emit_cfg_flags(); } + +fn link_to_conda() { + let hdf5_root = env::var("DEP_HDF5CONDA_ROOT").unwrap(); + println!("cargo:root={}", &hdf5_root); + let hdf5_incdir = env::var("DEP_HDF5CONDA_INCLUDE").unwrap(); + println!("cargo:include={}", &hdf5_incdir); + let hdf5_lib = env::var("DEP_HDF5CONDA_LIBRARY").unwrap(); + println!("cargo:library={}", &hdf5_lib); + + println!("cargo:rustc-link-search={}/lib", &hdf5_root); + println!("cargo:rustc-link-lib={}", &hdf5_lib); + + let header = Header::parse(&hdf5_incdir); + let config = Config { header, inc_dir: "".into(), link_paths: Vec::new() }; + config.emit_cfg_flags(); +} From 2ab9d009efea496af49ab15570f5e2b4b265a982 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 25 Nov 2021 20:48:36 +0000 Subject: [PATCH 2/8] Remove extra /lib --- hdf5-sys/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index 7bb603a6b..3b66dd353 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -691,7 +691,7 @@ fn link_to_conda() { let hdf5_lib = env::var("DEP_HDF5CONDA_LIBRARY").unwrap(); println!("cargo:library={}", &hdf5_lib); - println!("cargo:rustc-link-search={}/lib", &hdf5_root); + println!("cargo:rustc-link-search={}", &hdf5_root); println!("cargo:rustc-link-lib={}", &hdf5_lib); let header = Header::parse(&hdf5_incdir); From 3f56c97c7d2e34652056ed7fac009bca4d98ff02 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 25 Nov 2021 20:54:59 +0000 Subject: [PATCH 3/8] Add conda tests to CI --- .github/workflows/ci.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a2aef240..5a285ce70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -234,6 +234,27 @@ jobs: - name: Build and test all crates run: cargo test -vv + conda-download: + name: Conda forge ${{ matrix.os }} + runs-on: ${{ matrix.os }}-latest + strategy: + fail-fast: false + matrix: + include: + - {os: ubuntu, rust: stable} + - {os: windows, rust: stable-msvc} + - {os: windows, rust: stable-gnu} + - {os: macos, rust: stable} + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: {submodules: false} + - name: Install Rust (${{ matrix.rust }}) + uses: actions-rs/toolchain@v1 + with: {toolchain: '${{ matrix.rust }}', profile: minimal, override: true} + - name: Run tests + run: cargo test --features hdf5-sys/conda --workspace --exclude hdf5-derive,hdf5-src + msrv: name: Minimal Supported Rust Version runs-on: ubuntu-18.04 @@ -264,6 +285,7 @@ jobs: run: sudo apt-get update && sudo apt install wine64 mingw-w64 - name: Build and test run: env CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUNNER=wine64 cargo test --features hdf5-sys/static --target x86_64-pc-windows-gnu -- --skip test_compile_fail + addr_san: name: Address sanitizer runs-on: ubuntu-latest From c84cd102b4e94dfe65c9bf046dcefc54a8794057 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 25 Nov 2021 21:16:32 +0000 Subject: [PATCH 4/8] Replace macos library --- hdf5-conda/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hdf5-conda/build.rs b/hdf5-conda/build.rs index 638049ff4..ee9553ee3 100644 --- a/hdf5-conda/build.rs +++ b/hdf5-conda/build.rs @@ -75,8 +75,8 @@ const DOWNLOAD_BINARY: &str = { "linux-64/hdf5-1.12.1-nompi_h2750804_102.tar.bz2" } else if #[cfg(all(target_os = "linux", target_arch = "aarch64"))] { "linux-aarch64/hdf5-1.12.1-nompi_h774d4d8_102.tar.bz2" - } else if #[cfg(any(target_os = "osx", target_arch = "x86_64"))] { - "osx-64/hdf5-1.12.1-nompi_hd9e8a45_102.tar.bz2" + } else if #[cfg(any(target_os = "macos", target_arch = "x86_64"))] { + "osx-64/hdf5-1.12.1-nompi_h2f0ef1a_102.tar.bz2" } else if #[cfg(any(target_os = "windows", target_arch = "x86_64"))] { "hdf5-1.12.1-nompi_h2a0e4a3_102.tar.bz2" } else { From 1970c2c7b64ff4fa73257d7633c0ae0c60cebd63 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 25 Nov 2021 21:22:49 +0000 Subject: [PATCH 5/8] Add windows midfix for url --- hdf5-conda/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hdf5-conda/build.rs b/hdf5-conda/build.rs index ee9553ee3..d146cd2d0 100644 --- a/hdf5-conda/build.rs +++ b/hdf5-conda/build.rs @@ -78,7 +78,7 @@ const DOWNLOAD_BINARY: &str = { } else if #[cfg(any(target_os = "macos", target_arch = "x86_64"))] { "osx-64/hdf5-1.12.1-nompi_h2f0ef1a_102.tar.bz2" } else if #[cfg(any(target_os = "windows", target_arch = "x86_64"))] { - "hdf5-1.12.1-nompi_h2a0e4a3_102.tar.bz2" + "win-64/hdf5-1.12.1-nompi_h57737ce_102.tar.bz2" } else { compile_error!("This package can not be used on this arch"); } From f9dafc75bb870ac4bd8ee87a8b4338e2a0729362 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 25 Nov 2021 21:36:33 +0000 Subject: [PATCH 6/8] Print contents of inc_dir --- hdf5-sys/build.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hdf5-sys/build.rs b/hdf5-sys/build.rs index 3b66dd353..e94e2a9e6 100644 --- a/hdf5-sys/build.rs +++ b/hdf5-sys/build.rs @@ -217,6 +217,11 @@ impl Header { fn get_conf_header>(inc_dir: P) -> PathBuf { let inc_dir = inc_dir.as_ref(); + println!("{}", inc_dir.display()); + + for d in inc_dir.read_dir().unwrap() { + println!("{:?}", d.unwrap()); + } if inc_dir.join("H5pubconf.h").is_file() { inc_dir.join("H5pubconf.h") From 48fcdff9af94f815ce0214503f859fd2af571b39 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Thu, 25 Nov 2021 21:59:31 +0000 Subject: [PATCH 7/8] Use forward slash for win path --- hdf5-conda/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hdf5-conda/build.rs b/hdf5-conda/build.rs index d146cd2d0..437973316 100644 --- a/hdf5-conda/build.rs +++ b/hdf5-conda/build.rs @@ -38,7 +38,7 @@ const INC_PATH: &str = { if #[cfg(any(target_os = "linux", target_os = "macos"))] { "include" } else if #[cfg(target_os = "windows")] { - "Library\\include" + "Library/include" } else { compile_error!("This crate can not be used on this platform"); } @@ -62,7 +62,7 @@ const LIB_PATH: &str = { if #[cfg(any(target_os = "linux", target_os = "macos"))] { "lib" } else if #[cfg(target_os = "windows")] { - "Library\\lib" + "Library/lib" } else { compile_error!("This crate can not be used on this platform"); } From f5aeee80d6d9788863db125e94014f07d6822063 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Sat, 27 Nov 2021 12:06:23 +0100 Subject: [PATCH 8/8] Fix cfg flags --- hdf5-conda/build.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hdf5-conda/build.rs b/hdf5-conda/build.rs index 437973316..4ee110397 100644 --- a/hdf5-conda/build.rs +++ b/hdf5-conda/build.rs @@ -75,9 +75,9 @@ const DOWNLOAD_BINARY: &str = { "linux-64/hdf5-1.12.1-nompi_h2750804_102.tar.bz2" } else if #[cfg(all(target_os = "linux", target_arch = "aarch64"))] { "linux-aarch64/hdf5-1.12.1-nompi_h774d4d8_102.tar.bz2" - } else if #[cfg(any(target_os = "macos", target_arch = "x86_64"))] { + } else if #[cfg(all(target_os = "macos", target_arch = "x86_64"))] { "osx-64/hdf5-1.12.1-nompi_h2f0ef1a_102.tar.bz2" - } else if #[cfg(any(target_os = "windows", target_arch = "x86_64"))] { + } else if #[cfg(all(target_os = "windows", target_arch = "x86_64"))] { "win-64/hdf5-1.12.1-nompi_h57737ce_102.tar.bz2" } else { compile_error!("This package can not be used on this arch");