diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c647a95d42e..32be8206f0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -585,26 +585,26 @@ jobs: - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" + - name: Install Rust nightly toolchain run: | rustup toolchain install $NIGHTLY_TOOLCHAIN rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview - - name: Build FFI library (asan) - run: | - RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \ - -C debuginfo=2 -C opt-level=0 -C strip=none -Zexternal-clangrt \ - -Zsanitizer=address,leak" \ - cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \ - --target x86_64-unknown-linux-gnu -Zbuild-std \ - -p vortex-ffi - name: Build C++ tests and examples (asan) - run: | - mkdir -p build - cmake -S lang/cpp -Bbuild -DSANITIZER=asan -DBUILD_TESTS=1 -DBUILD_EXAMPLES=1 -DTARGET_TRIPLE="x86_64-unknown-linux-gnu" - cmake --build build --parallel $(nproc) + env: + RUSTUP_TOOLCHAIN: ${{ env.NIGHTLY_TOOLCHAIN }} + run: | + cmake -S lang/cpp -B build \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DVORTEX_SANITIZER=asan,lsan,ubsan \ + -DVORTEX_SANITIZE_RUST_STD=ON \ + -DVORTEX_BUILD_TESTING=ON \ + -DVORTEX_BUILD_EXAMPLES=ON + cmake --build build --parallel - name: Run C++ tests - run: | - build/tests/vortex_cxx_test + run: ctest --test-dir build --output-on-failure - name: Run C++ examples run: | cd build/examples @@ -782,13 +782,13 @@ jobs: - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" - - name: "regenerate FFI header file" - run: | - cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi - - name: Build and run C++ unit tests - run: | - cd vortex-ffi - mkdir build - cmake -Bbuild -DRUST_BUILD_PROFILE=ci - cmake --build build -j $(nproc) - ctest --test-dir build -j $(nproc) + - name: Build and run C API tests + env: + # Nightly regenerates cinclude/vortex.h with cbindgen during the build. + RUSTUP_TOOLCHAIN: ${{ env.NIGHTLY_TOOLCHAIN }} + run: | + cmake -S vortex-ffi -B vortex-ffi/build \ + -DVORTEX_CARGO_PROFILE=ci \ + -DVORTEX_BUILD_TESTING=ON + cmake --build vortex-ffi/build --parallel + ctest --test-dir vortex-ffi/build --output-on-failure diff --git a/.github/workflows/rust-instrumented.yml b/.github/workflows/rust-instrumented.yml index d186ce8dbac..9a97c6093e3 100644 --- a/.github/workflows/rust-instrumented.yml +++ b/.github/workflows/rust-instrumented.yml @@ -140,8 +140,6 @@ jobs: sudo apt-get update sudo apt-get install -y lcov libjson-xs-perl fi - - name: Build FFI library - run: cargo build -p vortex-ffi - name: Build and test C++ API with coverage working-directory: lang/cpp run: ./gcov-report.sh @@ -210,17 +208,14 @@ jobs: --target x86_64-unknown-linux-gnu -Zbuild-std \ -p vortex-buffer -p vortex-fastlanes -p vortex-fsst -p vortex-alp -p vortex-array - # vortex-ffi requires --no-default-features as otherwise we pull in - # Mimalloc which interferes with sanitizers # cargo nextest reports less sanitizer issues than cargo test - # TODO(myrrc): remove --no-default-features once we make Mimalloc opt-in # --tests skips doctests: rustdoc ignores RUSTFLAGS (it uses RUSTDOCFLAGS), # so doctests would build vortex-ffi without -Zsanitizer and mismatch the # sanitizer-built deps. - name: Run vortex-ffi tests with sanitizer run: | RUSTFLAGS="${RUSTFLAGS} ${{ matrix.sanitizer_flags }}" \ - cargo +$NIGHTLY_TOOLCHAIN test --locked --no-default-features \ + cargo +$NIGHTLY_TOOLCHAIN test --locked \ --target x86_64-unknown-linux-gnu --no-fail-fast -Zbuild-std \ -p vortex-ffi --tests -- --no-capture @@ -231,10 +226,8 @@ jobs: include: # We don't run memory sanitizer as it's clang-only and provides many # false positives for Catch2 - - sanitizer: asan - sanitizer_flags: "-Zsanitizer=address,leak" + - sanitizer: "asan,lsan,ubsan" - sanitizer: tsan - sanitizer_flags: "-Zsanitizer=thread" name: "Rust/C++ FFI tests (${{ matrix.sanitizer }})" timeout-minutes: 5 env: @@ -246,8 +239,6 @@ jobs: TSAN_OPTIONS: "symbolize=1:suppressions=${{ github.workspace }}/vortex-ffi/tsan_suppressions.txt" TSAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer" VORTEX_SKIP_SLOW_TESTS: "1" - # -Cunsafe-allow-abi-mismatch=sanitizer: libraries like compiler_builtins - # unset -Zsanitizer flag and we should allow that. runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id) @@ -265,20 +256,20 @@ jobs: run: | rustup toolchain install $NIGHTLY_TOOLCHAIN rustup component add --toolchain $NIGHTLY_TOOLCHAIN rust-src rustfmt clippy llvm-tools-preview - - name: Build FFI library - run: | - # TODO(myrrc): remove --no-default-features - RUSTFLAGS="-A warnings -Cunsafe-allow-abi-mismatch=sanitizer \ - -C debuginfo=2 -C opt-level=0 -C strip=none -Zexternal-clangrt \ - ${{ matrix.sanitizer_flags }}" \ - cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \ - --target x86_64-unknown-linux-gnu -Zbuild-std \ - -p vortex-ffi - - name: Build FFI library tests and examples + - name: Build FFI library, tests, and examples working-directory: vortex-ffi + env: + RUSTUP_TOOLCHAIN: ${{ env.NIGHTLY_TOOLCHAIN }} run: | - cmake -Bbuild -DBUILD_TESTS=1 -DBUILD_EXAMPLES=1 -DSANITIZER=${{ matrix.sanitizer }} -DTARGET_TRIPLE="x86_64-unknown-linux-gnu" - cmake --build build -j + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DVORTEX_SANITIZER=${{ matrix.sanitizer }} \ + -DVORTEX_SANITIZE_RUST_STD=ON \ + -DVORTEX_BUILD_TESTING=ON \ + -DVORTEX_BUILD_EXAMPLES=ON + cmake --build build --parallel - name: Run tests run: | set -o pipefail diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000000..683cef6f506 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +# CMake entry point for the Vortex native libraries. +# +# Layout: +# vortex-ffi/ Builds the Rust FFI archive with Cargo and exports +# Vortex::ffi_static for C consumers. Its cmake/ modules select +# the Rust toolchain, drive Cargo, and attach the system libraries +# the archive needs. The Cargo cache and the staged archive live +# under ffi/ in the build directory. +# lang/cpp/ Builds the C++ wrapper on top and exports Vortex::cpp_static. +# +# Usage: +# add_subdirectory(path/to/vortex vortex) +# target_link_libraries(my_c_target PRIVATE Vortex::ffi_static) +# target_link_libraries(my_cpp_target PRIVATE Vortex::cpp_static) +# +# Each subdirectory also configures on its own, and lang/cpp adds vortex-ffi +# itself when nothing else has. A separate `cargo build` is never required. +# +# Options, set before add_subdirectory() or passed with -D: +# VORTEX_BUILD_TESTING Tests of both layers, run with ctest. +# VORTEX_BUILD_EXAMPLES Examples of both layers. +# VORTEX_CARGO_PROFILE Cargo profile; otherwise mapped from CMAKE_BUILD_TYPE. +# VORTEX_ENABLE_CUDA Linux-only CUDA-enabled FFI archive. +# VORTEX_SANITIZER Comma-separated list of asan, lsan, ubsan, tsan +# for Rust, C, and C++ together; ubsan has no Rust +# side and covers C and C++ only. +# VORTEX_SANITIZE_RUST_STD Also instrument Rust's standard library. +# VORTEX_WARNINGS_AS_ERRORS ON for a Vortex-owned top level, OFF when embedded. +# +# A Vortex-owned top level, this file or either subdirectory, also defaults +# CMAKE_BUILD_TYPE to Debug. +# +# Documentation: +# lang/cpp/README.md Full option reference, embedding rules, the +# shared-library boundary, CUDA deployment, +# sanitizer and coverage workflows. +# docs/getting-started/cpp.rst C++ quickstart with API examples. +# vortex-ffi/README.md C API: runtime threading, header regeneration, +# CMake usage, Rust-side sanitizer tests. +# vortex-ffi/cinclude/vortex.h The C API header, generated by cbindgen. +# vortex-cuda/ffi/README.md CUDA-enabled FFI and Arrow Device export. +# vortex-cuda/README.md CUDA crate and cuDF integration notes. +# docs/project/bindings.md Language bindings roadmap. + +cmake_minimum_required(VERSION 3.28) + +project(VortexNative DESCRIPTION "Vortex native libraries" LANGUAGES NONE) + +# A standalone configure writes compile_commands.json for editor tooling. +if(PROJECT_IS_TOP_LEVEL) + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +endif() + +# ctest only descends into subdirectories from a root that has testing enabled. +enable_testing() + +add_subdirectory(vortex-ffi ffi) +add_subdirectory(lang/cpp cpp) diff --git a/Cargo.toml b/Cargo.toml index 3c0368e190e..5765f61b5c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -419,6 +419,10 @@ lto = "off" debug = "full" inherits = "release" +[profile.release_size] +inherits = "release" +opt-level = "z" + [profile.bench] codegen-units = 16 debug = "full" diff --git a/docs/getting-started/cpp.rst b/docs/getting-started/cpp.rst index fa56a41ffa6..ffd0d9c6572 100644 --- a/docs/getting-started/cpp.rst +++ b/docs/getting-started/cpp.rst @@ -10,39 +10,59 @@ The only dependency apart from Vortex is ``nanoarrow``. in using Vortex from C++ or you want a feature not covered yet e.g. extension support. -Installation ------------- +Building from source +-------------------- -We don't provide prebuilt library files (yet) so you will need to build Vortex -from source, and for that you will need: - -- C++20, -- Rust toolchain, -- and CMake 3.10. +Vortex does not provide prebuilt C++ libraries yet. Building from source requires +C++20, CMake 3.28 or newer with a single-config generator, and native Cargo and +rustc 1.95 or newer with the target standard library installed. The initial CMake +integration builds static position-independent-code libraries only. Use a complete +workspace checkout; the C++ directory cannot be built from an isolated source copy. .. code-block:: bash git clone --depth 1 https://github.com/vortex-data/vortex cd vortex - cargo build --release -p vortex-ffi - cmake -S lang/cpp -Bbuild -DCMAKE_BUILD_TYPE=Release - # To build the examples, pass -DBUILD_EXAMPLES=1 - # cmake -S lang/cpp -Bbuild -DBUILD_EXAMPLES=1 + cmake -S lang/cpp -B build/cpp -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DVORTEX_BUILD_EXAMPLES=ON + cmake --build build/cpp --parallel + +Configuration locates Cargo and rustc and takes the Rust target from the rustc +host. The locked ``vortex-ffi`` compilation occurs only when the CMake build runs, +so a separate ``cargo build`` step is neither required nor recommended. +Building examples can download nanoarrow during configure; Cargo can download +locked Rust dependencies during the build. - cmake --build build -j +Tests and examples default to ``OFF``. ``VORTEX_WARNINGS_AS_ERRORS`` defaults to +``ON`` for a standalone build and ``OFF`` when a parent adds ``lang/cpp``. -This produces a shared and a static library which you can use directly or via +Source-tree consumers add the repository root, which builds the Rust FFI once and +the C++ wrapper on top, and link the canonical target: .. code-block:: cmake - # static library - target_link_libraries(target PRIVATE vortex_cxx) - # shared library - target_link_libraries(target PRIVATE vortex_cxx_shared) + add_subdirectory(path/to/vortex vortex) + target_link_libraries(target PRIVATE Vortex::cpp_static) + +The Vortex archives are PIC and can be embedded into a shared parent. Keep calls +behind a private C++ translation unit and use the parent's normal version script or +exported-symbol allowlist: the shared parent owns its public ABI, and the Vortex +interface target does not apply parent-wide symbol-export policy. + +The CMake integration is source-only: it does not provide installation rules or a +``find_package(Vortex)`` package. Downstream projects should vendor or fetch a pinned +Vortex checkout and add it directly. + +Native macOS arm64 is supported for standalone development, but macOS is not a +cuDF integration target. GNU/Linux x86_64 and aarch64 are modeled, while full +GCC 14, Conda compiler-wrapper, glibc 2.28, and cuDF validation remains deferred. -Have a look at the `examples -`_ +See the `C++ bindings README +`_ for all CMake +options, cache behavior, and source-integration rules. Have a look at the +`examples `_ directory as well. Reading files @@ -205,7 +225,7 @@ Now you can build the example and read back the generated files: .. code-block:: - ./build/examples/writer people0.vortex - ./build/examples/writer people1.vortex - ./build/examples/writer me.vortex - ./build/examples/reader + ./build/cpp/examples/writer people0.vortex + ./build/cpp/examples/writer people1.vortex + ./build/cpp/examples/writer me.vortex + ./build/cpp/examples/reader diff --git a/docs/project/bindings.md b/docs/project/bindings.md index 3311a7a3663..d2d7dc7f922 100644 --- a/docs/project/bindings.md +++ b/docs/project/bindings.md @@ -91,7 +91,8 @@ Migrate from cxx to wrapping the C API: - **RAII wrappers:** Provide C++ classes that manage lifetime of C API objects (files, scanners, arrays, streams). -- **CMake integration:** Ship a CMake config so downstream projects can `find_package(Vortex)`. +- **CMake integration:** Provide source-tree CMake targets so vendored downstream projects can + build Vortex with `add_subdirectory`. - **Header generation:** Auto-generate C++ headers from the C API headers, adding type safety and namespace scoping. - **Target:** Tier 2 (native array access through the C API). diff --git a/lang/cpp/CMakeLists.txt b/lang/cpp/CMakeLists.txt index 59beccb5cac..71192e3b754 100644 --- a/lang/cpp/CMakeLists.txt +++ b/lang/cpp/CMakeLists.txt @@ -1,159 +1,118 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -cmake_minimum_required(VERSION 3.10) -project(VortexCXX - VERSION 0.0.1 - LANGUAGES CXX) - -include(FetchContent) - -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED ON) - -find_program(SCCACHE_PROGRAM sccache) -if (SCCACHE_PROGRAM) - set(CMAKE_C_COMPILER_LAUNCHER "${SCCACHE_PROGRAM}") - set(CMAKE_CXX_COMPILER_LAUNCHER "${SCCACHE_PROGRAM}") - message(STATUS "Sccache found: ${SCCACHE_PROGRAM}") -else () - message(STATUS "Sccache not found") -endif () - -option(BUILD_TESTS "Build tests" OFF) - -set(SANITIZER "" CACHE STRING "Build with sanitizers") -set(TARGET_TRIPLE "" CACHE STRING "Rust target triple for FFI library") -set(RUST_BUILD_PROFILE "" CACHE STRING "Cargo profile name for Rust FFI library") - -if (NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Debug) -endif() - -if (NOT SANITIZER STREQUAL "") - message(NOTICE "Sanitizer: ${SANITIZER}") - if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - message(FATAL_ERROR "Only debug build is supported for sanitizer builds") - endif() - - if (SANITIZER STREQUAL "asan") - set(SANITIZER_FLAGS "-fsanitize=address,undefined,leak") - elseif (SANITIZER STREQUAL "tsan") - set(SANITIZER_FLAGS "-fsanitize=thread") - else() - message(FATAL_ERROR "Unknown sanitizer ${SANITIZER}") - endif() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SANITIZER_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SANITIZER_FLAGS}") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${SANITIZER_FLAGS}") -endif() - -if (RUST_BUILD_PROFILE STREQUAL "") - if (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") - set(RUST_BUILD_PROFILE "release_debug") - else() - string(TOLOWER "${CMAKE_BUILD_TYPE}" RUST_BUILD_PROFILE) - endif() +# Entry point for the static Vortex C++ wrapper. It consumes the Rust FFI +# archive from vortex-ffi and exports Vortex::cpp_static. It supports standalone +# configuration and add_subdirectory() from the repository root or a parent. + +cmake_minimum_required(VERSION 3.28) + +# project() is safe inside a parent's add_subdirectory(). A Vortex-owned top +# level, this directory or the repository root, gets developer defaults below; +# an embedding parent keeps its own. +project(VortexCXX DESCRIPTION "Vortex C++ wrapper" LANGUAGES C CXX) +if(CMAKE_PROJECT_NAME MATCHES "^Vortex") + set(_vortex_top_level ON) +else() + set(_vortex_top_level OFF) endif() -set(VORTEX_FFI_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../vortex-ffi") -set(VORTEX_FFI_LIB_DIR "${VORTEX_FFI_DIR}/../target/${TARGET_TRIPLE}/${RUST_BUILD_PROFILE}") -set(VORTEX_FFI_HEADERS "${VORTEX_FFI_DIR}/cinclude") +# Vortex does not use C++ modules, so it does not require compiler-specific +# dependency scanners such as clang-scan-deps. +set(CMAKE_CXX_SCAN_FOR_MODULES OFF) -if(APPLE) - set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.a") - set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.dylib") -elseif(WIN32) - set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.lib") - set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.dll") -else() - set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.a") - set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.so") +# A standalone configure writes compile_commands.json for editor tooling. +if(PROJECT_IS_TOP_LEVEL) + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) endif() -if(NOT EXISTS "${VORTEX_FFI_STATIC}") - message(FATAL_ERROR - "vortex-ffi static library not found at ${VORTEX_FFI_STATIC}. " - "Run: cargo build --profile -p vortex-ffi") +if(_vortex_top_level AND NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Debug) endif() -add_library(vortex_ffi STATIC IMPORTED) -set_target_properties(vortex_ffi PROPERTIES - IMPORTED_LOCATION "${VORTEX_FFI_STATIC}" - INTERFACE_INCLUDE_DIRECTORIES "${VORTEX_FFI_HEADERS}") - -if(EXISTS "${VORTEX_FFI_SHARED}") - add_library(vortex_ffi_shared SHARED IMPORTED) - set_target_properties(vortex_ffi_shared PROPERTIES - IMPORTED_LOCATION "${VORTEX_FFI_SHARED}" - INTERFACE_INCLUDE_DIRECTORIES "${VORTEX_FFI_HEADERS}" - INTERFACE_LINK_OPTIONS "LINKER:-rpath,${VORTEX_FFI_LIB_DIR}") +# Public options. Embedding parents set these before add_subdirectory(). The +# Cargo, CUDA, and sanitizer options are defined by vortex-ffi/CMakeLists.txt. +option(VORTEX_BUILD_TESTING "Build Vortex tests" OFF) +option(VORTEX_BUILD_EXAMPLES "Build Vortex examples" OFF) +option(VORTEX_WARNINGS_AS_ERRORS + "Treat warnings in Vortex-owned sources as errors" + "${_vortex_top_level}") + +# The Rust FFI archive comes from vortex-ffi. A standalone configure of this +# directory adds it here; the repository root or a parent may already have. +if(NOT TARGET Vortex::ffi_static) + add_subdirectory(../../vortex-ffi ffi) endif() -file(GLOB VORTEX_CXX_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp") -add_library(vortex_cxx STATIC ${VORTEX_CXX_SOURCES}) -target_include_directories(vortex_cxx PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${CMAKE_CURRENT_SOURCE_DIR}/../vortex-ffi/cinclude -) -target_link_libraries(vortex_cxx PUBLIC vortex_ffi) -add_library(vortex::cxx ALIAS vortex_cxx) +# The C++ wrapper over the C FFI. Public headers live under include/. +file(GLOB _vortex_cpp_sources CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp") + +# Consumers inherit the FFI archive, its headers, and its system libraries +# through this target. Hidden visibility keeps wrapper symbols out of a parent +# shared library's exports. +add_library(vortex_cxx STATIC ${_vortex_cpp_sources}) +set_target_properties(vortex_cxx PROPERTIES + POSITION_INDEPENDENT_CODE ON + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON) target_compile_features(vortex_cxx PUBLIC cxx_std_20) - -set(APPLE_LINK_FLAGS "-framework CoreFoundation -framework Security") - -if(APPLE) - target_link_libraries(vortex_cxx PRIVATE ${APPLE_LINK_FLAGS}) +target_include_directories(vortex_cxx PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") +target_link_libraries(vortex_cxx PUBLIC Vortex::ffi_static) + +# Warning policy is private to Vortex-owned sources so embedding this target +# never injects warning flags into consumers. +target_compile_options(vortex_cxx PRIVATE -Wall -Wextra -Wpedantic) +if(VORTEX_WARNINGS_AS_ERRORS) + target_compile_options(vortex_cxx PRIVATE -Werror) endif() -if(TARGET vortex_ffi_shared) - add_library(vortex_cxx_shared SHARED ${VORTEX_CXX_SOURCES}) - set_target_properties(vortex_cxx_shared PROPERTIES POSITION_INDEPENDENT_CODE ON) - target_include_directories(vortex_cxx_shared PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${CMAKE_CURRENT_SOURCE_DIR}/../vortex-ffi/cinclude - ) - target_link_libraries(vortex_cxx_shared PUBLIC vortex_ffi_shared) - add_library(vortex::cxx_shared ALIAS vortex_cxx_shared) - target_compile_features(vortex_cxx_shared PUBLIC cxx_std_20) - - if(APPLE) - target_link_libraries(vortex_cxx_shared PRIVATE ${APPLE_LINK_FLAGS}) - endif() -endif() +# The only supported consumer-facing name for the C++ API. +add_library(Vortex::cpp_static ALIAS vortex_cxx) + +# Test and example dependencies are fetched at configure time and never installed. +if(VORTEX_BUILD_TESTING OR VORTEX_BUILD_EXAMPLES) + include(FetchContent) -if (BUILD_TESTS OR BUILD_EXAMPLES) + # Nanoarrow's developer warnings use -Werror and can reject newer compilers. + # Its headers are system headers when consumed by Vortex-owned targets. + set(NANOARROW_DEBUG_EXTRA_WARNINGS OFF CACHE BOOL + "Enable Nanoarrow's additional compiler warnings") FetchContent_Declare( - Nanoarrow - GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow - GIT_TAG apache-arrow-nanoarrow-0.8.0 - ) + Nanoarrow + SYSTEM + GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow + GIT_TAG apache-arrow-nanoarrow-0.8.0) FetchContent_MakeAvailable(Nanoarrow) - # arrow-nanoarrow creates a self-referential symlink - # python/subprojects/arrow-nanoarrow -> ../.. that makes any recursive - if (IS_SYMLINK "${nanoarrow_SOURCE_DIR}/python/subprojects/arrow-nanoarrow") + # arrow-nanoarrow creates a self-referential symlink: + # python/subprojects/arrow-nanoarrow -> ../.. + if(IS_SYMLINK "${nanoarrow_SOURCE_DIR}/python/subprojects/arrow-nanoarrow") file(REMOVE "${nanoarrow_SOURCE_DIR}/python/subprojects/arrow-nanoarrow") endif() endif() -if (BUILD_TESTS) +if(VORTEX_BUILD_TESTING) FetchContent_Declare( Catch + SYSTEM GIT_REPOSITORY https://github.com/catchorg/Catch2.git - GIT_TAG v3.8.1 - ) + GIT_TAG v3.8.1) FetchContent_MakeAvailable(Catch) - include(Catch) + # By path: Catch2 exports its module directory only to the directory that + # populated it, which may be vortex-ffi/test. + include("${catch_SOURCE_DIR}/extras/Catch.cmake") + + # Catch2 enables std::string_view, optional, variant, and byte support from + # C++17, in headers and compiled sources alike; build it with those on so it + # matches the C++23 tests that include its headers. + target_compile_features(Catch2 PRIVATE cxx_std_17) target_compile_definitions(Catch2 PRIVATE CATCH_CONFIG_NO_POSIX_SIGNALS) - include(CTest) + # Embedding parents must also call enable_testing() for ctest to find these. + enable_testing() add_subdirectory(tests) endif() -if (BUILD_EXAMPLES) +if(VORTEX_BUILD_EXAMPLES) add_subdirectory(examples) endif() diff --git a/lang/cpp/README.md b/lang/cpp/README.md index fe9d2527e63..6820a3dac8f 100644 --- a/lang/cpp/README.md +++ b/lang/cpp/README.md @@ -1,45 +1,181 @@ # Vortex C++ bindings -For a usage guide, see docs/api/cpp/index.rst. +Vortex provides a C++20 API for reading and writing Vortex files. See the +[C++ quickstart](../../docs/getting-started/cpp.rst) for API examples. -## Requirements +## Quick start -- CMake 3.10+ -- C++20 compiler (C++23 compiler for tests). -- Rust toolchain for building `vortex-ffi`. +Build from the repository root with CMake 3.28 or newer, native C/C++ compilers, and Cargo and +rustc available to CMake's program search: -## Build +```sh +cmake -S lang/cpp -B build/cpp -DCMAKE_BUILD_TYPE=Release +cmake --build build/cpp --parallel +``` + +CMake builds the Rust FFI through Cargo. A separate `cargo build` is not required. + +## Embed in a CMake project + +Vendor or fetch a pinned, complete Vortex checkout, then add the repository root once: + +```cmake +add_subdirectory(path/to/vortex vortex) +target_link_libraries(my_cpp_target PRIVATE Vortex::cpp_static) +target_link_libraries(my_c_target PRIVATE Vortex::ffi_static) +``` + +The root builds the Rust FFI archive once under `vortex-ffi` and layers this directory on top; +`lang/cpp` and `vortex-ffi` can also be added on their own. `Vortex::cpp_static` is an alias of the +`vortex_cxx` build target. It transitively links the FFI archive and required native libraries, so +consume the target rather than copying `libvortex_cxx.a` alone. + +The integration is source-only: it does not install Vortex or provide `find_package(Vortex)`. It +enables C and C++ when needed, but keeps compile and link policy target-scoped and does not replace +the parent's build type, language standards, compiler flags, linker flags, or `BUILD_SHARED_LIBS`. + +### Shared-library boundary + +Vortex can be linked privately into a shared library such as `libcudf`. Keep Vortex calls in private +C++ implementation files and preserve the parent's symbol-export policy. The parent must prevent +`vx_*` and other implementation symbols from becoming part of its public ABI, for example with an +ELF version script and `--exclude-libs,ALL` or a macOS exported-symbol allowlist. CUDA-enabled builds +also have runtime-library deployment requirements described below. + +## Supported configurations + +The build supports: + +- native GNU/Linux on x86_64 and aarch64; +- native macOS on arm64 for standalone development; +- a single-config generator; standalone builds default to `Debug`, and unless + `VORTEX_CARGO_PROFILE` is set, `Debug`, `Release`, `RelWithDebInfo`, and `MinSizeRel` map to + Cargo `dev`, `release`, `release_debug`, and `release_size`, respectively, an empty build type + uses `dev`, and other build types warn and use `dev`; and +- Cargo and rustc 1.95 or newer, which Cargo enforces from the workspace `rust-version` during + the build. + +CMake discovers Cargo and rustc with `find_program`; set `VORTEX_CARGO_EXECUTABLE` and +`VORTEX_RUSTC_EXECUTABLE` to override them. Rustup proxies run from the Vortex workspace, so they +honor its `rust-toolchain.toml`, and the `RUSTUP_TOOLCHAIN` value present at configure time applies +to every Cargo build until CMake is reconfigured. The rustc host selects the Rust target. + +Cross-compilation, Apple universal binaries, Windows, musl, multi-config generators, and shared +Vortex targets are not supported. Ninja is recommended. macOS is not a supported cuDF integration +target. + +## Configuration + +Set these public Vortex-specific options before `add_subdirectory`, or pass them with `-D` for a +standalone build. The Cargo, CUDA, and sanitizer options are defined by `vortex-ffi/CMakeLists.txt` +and apply to both layers: + +- `VORTEX_BUILD_TESTING=ON` builds the `vortex_cxx_test` C++23 test target and the C API tests in + `vortex-ffi`. Embedded builds require the parent to enable CTest. Default: `OFF`. +- `VORTEX_BUILD_EXAMPLES=ON` builds the C++ and C examples. Default: `OFF`. +- `VORTEX_ENABLE_CUDA=ON` selects the Linux-only `vortex-cuda-ffi` archive, adds `vortex_cuda.h` to + the existing `Vortex::cpp_static` target, and requires `find_package(CUDAToolkit)`. Set + `CUDAToolkit_ROOT` when CMake cannot find the toolkit. Default: `OFF`. +- `VORTEX_CARGO_PROFILE` overrides the Cargo profile inferred from `CMAKE_BUILD_TYPE`. Custom + profiles use the same-named Cargo artifact directory; Cargo's `test` and `bench` profiles are not + supported. Default: empty. +- `VORTEX_WARNINGS_AS_ERRORS` promotes warnings only while compiling `vortex_cxx`; it does not affect + tests, examples, consumers, Rust, or CUDA compilation. Default: `ON` standalone and `OFF` when + embedded. +- `VORTEX_SANITIZER` takes a comma-separated list of `asan`, `lsan`, `ubsan`, and `tsan`, for + example `asan,ubsan`. It requires `Debug` and Clang or AppleClang. Each sanitizer instruments the + C and C++ code, including Cargo-built C dependencies, and all but `ubsan` also instrument the Rust + code, since rustc has no UBSan. Rust instrumentation selects rustup's `nightly` toolchain unless + `RUSTUP_TOOLCHAIN` is set. Apple clang does not support `lsan` on arm64. Flags + propagate to targets linking Vortex, but CUDA device code, the CUB helper, and nvCOMP are not + sanitizer-instrumented. Default: empty. +- `VORTEX_SANITIZE_RUST_STD=ON` rebuilds Rust's standard library with the selected sanitizer and + requires the nightly `rust-src` component. Default: `OFF`. + +Cargo runs with the lockfile and the selected native target and profile. Optional features such as +`mimalloc` are not enabled. The integration supplies its complete Rust flag sequence, overriding Rust +flags from the environment and Cargo configuration. + +Cargo's target cache is stored under the `vortex-ffi` binary directory, which is `ffi/cargo-target` +inside a root or `lang/cpp` build directory. The Cargo target runs whenever +Vortex is built, while Cargo decides whether recompilation is needed. The standard CMake clean target +removes both CMake outputs and this Cargo cache: ```sh -cargo build --release -p vortex-ffi -cmake -Bbuild -DCMAKE_BUILD_TYPE=Release -cmake --build build -j +cmake --build build/cpp --target clean ``` -This will generate `libvortex_cxx` shared and static libraries. -You can use `vortex_cxx` and `vortex_cxx_shared` CMake targets. +Public headers are consumed from the source checkout. `vortex.h` and `vortex_cuda.h` are checked in. +A non-sanitizer nightly build may regenerate `vortex.h` with cbindgen and attempt to format it with +`clang-format`; stable and sanitizer builds leave it unchanged. CUDA builds also generate kernel +sources and PTX in the source checkout, so nightly and CUDA builds require it to be writable. + +Cargo may download locked dependencies during the build. CUDA builds additionally require libclang +for bindgen and may download the pinned CUDA 12 nvCOMP SDK directly from NVIDIA. Tests and examples +may download Nanoarrow, Catch2, and magic_enum during CMake configure. + +### CUDA deployment + +CUDA builds continue to use `Vortex::cpp_static`; no separate CUDA CMake target is created. Current +CUDA build scripts invoke NVCC with `-arch=native`, so `CMAKE_CUDA_ARCHITECTURES` has no effect. +Generated PTX is embedded in the Rust archive, but CMake does not stage `libvortex_cub.so` or the +downloaded `libnvcomp.so`. The CUB helper must remain at its original Cargo build path or be placed +next to the host executable; nvCOMP is loaded from its original Cargo build path. CUDA operations +also require a compatible NVIDIA driver and accessible GPU. Consequently, CUDA-enabled output is +not currently a self-contained relocatable deployment artifact. -## Test +## Development + +### Tests + +```sh +cmake -S lang/cpp -B build/cpp-tests -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DVORTEX_BUILD_TESTING=ON +cmake --build build/cpp-tests --parallel +ctest --test-dir build/cpp-tests --output-on-failure +``` + +### Examples ```sh -cargo build --release -p vortex-ffi -cmake -Bbuild -DBUILD_TESTS=ON -cmake --build build -j -ctest --test-dir build -j "$(nproc)" +cmake -S lang/cpp -B build/cpp-examples -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DVORTEX_BUILD_EXAMPLES=ON +cmake --build build/cpp-examples --parallel ``` -## Run examples +The `reader`, `writer`, `dtype`, `scan`, and `scan_to_arrow` executables are written to +`build/cpp-examples/examples/`. + +### Sanitizers + +Sanitizer builds use rustup's `nightly` toolchain unless `RUSTUP_TOOLCHAIN` selects another, and +CMake must use Clang or AppleClang: ```sh -cmake -Bbuild -DBUILD_EXAMPLES=ON -cmake --build build -j -./build/examples/hello-vortex +rustup toolchain install nightly +cmake -S lang/cpp -B build/cpp-asan -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DVORTEX_SANITIZER=asan,ubsan \ + -DVORTEX_BUILD_TESTING=ON +cmake --build build/cpp-asan --parallel +ctest --test-dir build/cpp-asan --output-on-failure ``` -## Check coverage +To also instrument Rust's standard library, install `rust-src` and configure with +`-DVORTEX_SANITIZE_RUST_STD=ON`. + +### Coverage -This will generate an LCOV directory `coverage`: +Run one of the following from `lang/cpp`. The helper reports C++ coverage only and requires a +compatible gcov/LCOV toolchain, plus `genhtml` for HTML output: ```sh -./gcov-report.sh generate +cd lang/cpp +./gcov-report.sh # Writes coverage.info +# Or: +./gcov-report.sh html # Also writes coverage/ ``` diff --git a/lang/cpp/examples/CMakeLists.txt b/lang/cpp/examples/CMakeLists.txt index 2733b745b96..c51777c08f6 100644 --- a/lang/cpp/examples/CMakeLists.txt +++ b/lang/cpp/examples/CMakeLists.txt @@ -1,7 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -foreach(name reader writer dtype scan scan_to_arrow) - add_executable(${name} ${name}.cpp) - target_link_libraries(${name} PRIVATE vortex_cxx_shared nanoarrow_shared) +# Target names carry a prefix so a build that also includes the C examples has +# no clashes; the executables keep their plain names. +foreach(_name reader writer dtype scan scan_to_arrow) + add_executable(cpp_${_name} ${_name}.cpp) + set_target_properties(cpp_${_name} PROPERTIES OUTPUT_NAME ${_name}) + target_link_libraries(cpp_${_name} PRIVATE Vortex::cpp_static nanoarrow_shared) endforeach() diff --git a/lang/cpp/examples/reader.cpp b/lang/cpp/examples/reader.cpp index 5cf331dffc5..a447ec6917b 100644 --- a/lang/cpp/examples/reader.cpp +++ b/lang/cpp/examples/reader.cpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +#include #include #include diff --git a/lang/cpp/gcov-report.sh b/lang/cpp/gcov-report.sh index 6c1365df57a..c9f982284fb 100755 --- a/lang/cpp/gcov-report.sh +++ b/lang/cpp/gcov-report.sh @@ -3,16 +3,23 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors +# Builds the C++ tests with gcov instrumentation, runs them, and writes +# coverage.info next to this script; pass `html` to also render coverage/. set -eu -cmake -Bbuild -DBUILD_TESTS=1 -DCMAKE_CXX_FLAGS='-fprofile-arcs -ftest-coverage' -cmake --build build -j +cd "$(dirname "$0")" + +cmake -S . -B build \ + -DVORTEX_BUILD_TESTING=ON \ + -DCMAKE_CXX_FLAGS=--coverage +cmake --build build --parallel ctest --test-dir build --output-on-failure -geninfo build/CMakeFiles/vortex_cxx_shared.dir/ \ +# lcov matches exclude globs against full source paths. +geninfo build/CMakeFiles/vortex_cxx.dir/ \ build/tests/CMakeFiles/vortex_cxx_test.dir/ \ --rc geninfo_unexecuted_blocks=1 \ - --exclude /usr --exclude build/_deps --exclude tests \ + --exclude '/usr/*' --exclude '*/_deps/*' --exclude '*/tests/*' \ -j -b src -o coverage.info -if [ $# -gt 0 ]; then +if [ "${1:-}" = html ]; then genhtml coverage.info -o coverage fi diff --git a/lang/cpp/include/vortex/common.hpp b/lang/cpp/include/vortex/common.hpp index cc8559e96c6..51f76190f0a 100644 --- a/lang/cpp/include/vortex/common.hpp +++ b/lang/cpp/include/vortex/common.hpp @@ -17,7 +17,6 @@ struct float16_t { constexpr friend bool operator==(float16_t, float16_t) = default; // NOLINTNEXTLINE constexpr operator float() const { - float result; const uint32_t sign = (bits >> 15) & 1; const uint32_t exponent = (bits >> 10) & 0x1F; const uint32_t mantissa = bits & 0x3FF; diff --git a/lang/cpp/tests/CMakeLists.txt b/lang/cpp/tests/CMakeLists.txt index 16a6e0290b9..02d8327e4df 100644 --- a/lang/cpp/tests/CMakeLists.txt +++ b/lang/cpp/tests/CMakeLists.txt @@ -1,22 +1,21 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -# need this for std::float16_t -set(CMAKE_CXX_STANDARD 23) - FetchContent_Declare( magic_enum + SYSTEM GIT_REPOSITORY https://github.com/Neargye/magic_enum.git - GIT_TAG v0.9.7 -) + GIT_TAG v0.9.7) FetchContent_MakeAvailable(magic_enum) file(GLOB TEST_FILES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") add_executable(vortex_cxx_test ${TEST_FILES}) +# Required by the std::float16_t coverage. +target_compile_features(vortex_cxx_test PRIVATE cxx_std_23) target_include_directories(vortex_cxx_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(vortex_cxx_test PRIVATE - vortex_cxx_shared + Vortex::cpp_static Catch2::Catch2WithMain magic_enum::magic_enum nanoarrow_shared) diff --git a/vortex-ffi/CMakeLists.txt b/vortex-ffi/CMakeLists.txt index 05ca5c348c5..bcdb8fc2897 100644 --- a/vortex-ffi/CMakeLists.txt +++ b/vortex-ffi/CMakeLists.txt @@ -1,127 +1,88 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -cmake_minimum_required(VERSION 3.10) -include(FetchContent) +# Entry point for the Rust FFI archive. It builds vortex-ffi with Cargo and +# exports Vortex::ffi_static for C consumers; lang/cpp layers the C++ wrapper on +# top. It supports standalone configuration and add_subdirectory() from a parent. -project(VortexFFI - VERSION 0.0.1 - LANGUAGES C) -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -set(CMAKE_C_STANDARD 17) -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror -Wextra -Wpedantic") +cmake_minimum_required(VERSION 3.28) -option(BUILD_TESTS "Build tests" OFF) -option(BUILD_EXAMPLES "Build examples" OFF) - -set(SANITIZER "" CACHE STRING "Build with sanitizers") -set(TARGET_TRIPLE "" CACHE STRING "Rust target triple for FFI library") -set(RUST_BUILD_PROFILE "" CACHE STRING "Cargo profile name for the Rust FFI library") - -if (NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Debug) -endif() - -if (NOT SANITIZER STREQUAL "") - message(NOTICE "Sanitizer: ${SANITIZER}") - if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - message(FATAL_ERROR "Only debug build is supported for sanitizer builds") - endif() - - if (SANITIZER STREQUAL "asan") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address,undefined,leak") - if (BUILD_TESTS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address,undefined,leak") - endif() - elseif (SANITIZER STREQUAL "tsan") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=thread") - if (BUILD_TESTS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread") - endif() - else() - message(FATAL_ERROR "Unknown sanitizer ${SANITIZER}") - endif() -endif() - -message(NOTICE "Build type: ${CMAKE_BUILD_TYPE}") -if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug" AND - NOT CMAKE_BUILD_TYPE STREQUAL "Release" AND - NOT CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") - message(FATAL_ERROR "vortex-ffi can only be built in Release, Debug, or RelWithDebInfo mode") -endif() - -if (RUST_BUILD_PROFILE STREQUAL "") - if (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") - set(RUST_BUILD_PROFILE "release_debug") - else() - string(TOLOWER ${CMAKE_BUILD_TYPE} RUST_BUILD_PROFILE) - endif() -endif() -set(LIBRARY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../target/${TARGET_TRIPLE}/${RUST_BUILD_PROFILE}") - -if(WIN32) - set(LIBRARY_PATH "${LIBRARY_DIR}/libvortex_ffi.lib") - set(LIBRARY_PATH_SHARED "${LIBRARY_DIR}/libvortex_ffi.dll") -elseif(APPLE) - set(LIBRARY_PATH "${LIBRARY_DIR}/libvortex_ffi.a") - set(LIBRARY_PATH_SHARED "${LIBRARY_DIR}/libvortex_ffi.dylib") +# project() is safe inside a parent's add_subdirectory(). A Vortex-owned top +# level, this directory or the repository root, gets developer defaults below; +# an embedding parent keeps its own. +project(VortexFFI DESCRIPTION "Vortex C FFI" LANGUAGES C CXX) +if(CMAKE_PROJECT_NAME MATCHES "^Vortex") + set(_vortex_top_level ON) else() - set(LIBRARY_PATH "${LIBRARY_DIR}/libvortex_ffi.a") - set(LIBRARY_PATH_SHARED "${LIBRARY_DIR}/libvortex_ffi.so") + set(_vortex_top_level OFF) endif() -set(LIBRARY_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/cinclude") - -message(NOTICE " -Library dir ${LIBRARY_DIR} -Shared library path ${LIBRARY_PATH_SHARED} -Static library path ${LIBRARY_PATH} -Headers path ${LIBRARY_HEADERS}") +# Vortex does not use C++ modules, so it does not require compiler-specific +# dependency scanners such as clang-scan-deps. +set(CMAKE_CXX_SCAN_FOR_MODULES OFF) -if (NOT EXISTS "${LIBRARY_PATH_SHARED}") - message(FATAL_ERROR "Shared library not found, run `cargo build --release -p vortex-ffi`") +# A standalone configure writes compile_commands.json for editor tooling. +if(PROJECT_IS_TOP_LEVEL) + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) endif() -if (NOT EXISTS "${LIBRARY_PATH}") - message(FATAL_ERROR "Static library not found, run `cargo build --release -p vortex-ffi`") -endif() - -add_library(vortex_ffi STATIC IMPORTED) -set_target_properties(vortex_ffi PROPERTIES - IMPORTED_LOCATION "${LIBRARY_PATH}" - INTERFACE_INCLUDE_DIRECTORIES "${LIBRARY_HEADERS}" -) -add_library(vortex_ffi_shared SHARED IMPORTED) -set_target_properties(vortex_ffi_shared PROPERTIES - IMPORTED_LOCATION "${LIBRARY_PATH_SHARED}" - INTERFACE_INCLUDE_DIRECTORIES "${LIBRARY_HEADERS}" - INTERFACE_LINK_OPTIONS "LINKER:-rpath,${LIBRARY_DIR}" -) +if(_vortex_top_level AND NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Debug) +endif() -if (BUILD_TESTS OR BUILD_EXAMPLES) +# Public options. Embedding parents set these before add_subdirectory(). The +# testing and examples options are shared with lang/cpp. +option(VORTEX_BUILD_TESTING "Build Vortex tests" OFF) +option(VORTEX_BUILD_EXAMPLES "Build Vortex examples" OFF) +option(VORTEX_ENABLE_CUDA "Build the CUDA-enabled Vortex FFI" OFF) +set(VORTEX_CARGO_PROFILE "" CACHE STRING "Override the Cargo profile selected from CMAKE_BUILD_TYPE") + +# Sanitizers instrument the Rust archive, its C dependencies, and every target +# that links Vortex::ffi_static; Configure.cmake enforces the toolchain constraints. +set(VORTEX_SANITIZER "" CACHE STRING + "Comma-separated sanitizers for Vortex and its consumers: asan, lsan, ubsan, tsan") +option(VORTEX_SANITIZE_RUST_STD + "Rebuild Rust's standard library with sanitizer instrumentation" + OFF) + +option(VORTEX_WARNINGS_AS_ERRORS + "Treat warnings in Vortex-owned sources as errors" + "${_vortex_top_level}") + +# Define the Cargo build and the imported vortex_ffi_static archive. +include(cmake/Configure.cmake) + +# The only supported consumer-facing name for the C API. +add_library(Vortex::ffi_static ALIAS vortex_ffi_static) + +# Test and example dependencies are fetched at configure time and never installed. +if(VORTEX_BUILD_TESTING OR VORTEX_BUILD_EXAMPLES) + include(FetchContent) + + # Nanoarrow's developer warnings use -Werror and can reject newer compilers. + # Its headers are system headers when consumed by Vortex-owned targets. + set(NANOARROW_DEBUG_EXTRA_WARNINGS OFF CACHE BOOL + "Enable Nanoarrow's additional compiler warnings") FetchContent_Declare( - Nanoarrow - GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow - GIT_TAG apache-arrow-nanoarrow-0.8.0 - ) + Nanoarrow + SYSTEM + GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow + GIT_TAG apache-arrow-nanoarrow-0.8.0) FetchContent_MakeAvailable(Nanoarrow) - # arrow-nanoarrow creates a self-referential symlink + # arrow-nanoarrow creates a self-referential symlink: # python/subprojects/arrow-nanoarrow -> ../.. - if (IS_SYMLINK "${nanoarrow_SOURCE_DIR}/python/subprojects/arrow-nanoarrow") + if(IS_SYMLINK "${nanoarrow_SOURCE_DIR}/python/subprojects/arrow-nanoarrow") file(REMOVE "${nanoarrow_SOURCE_DIR}/python/subprojects/arrow-nanoarrow") endif() endif() -if (BUILD_TESTS) - enable_language(CXX) - set(CMAKE_CXX_STANDARD 20) - set(CMAKE_CXX_STANDARD_REQUIRED ON) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror -Wextra -Wpedantic") +if(VORTEX_BUILD_TESTING) + # Embedding parents must also call enable_testing() for ctest to find these. enable_testing() add_subdirectory(test) endif() -if (BUILD_EXAMPLES) +if(VORTEX_BUILD_EXAMPLES) add_subdirectory(examples) endif() diff --git a/vortex-ffi/README.md b/vortex-ffi/README.md index 5cac7b7f490..2acf080d6a9 100644 --- a/vortex-ffi/README.md +++ b/vortex-ffi/README.md @@ -27,100 +27,76 @@ cargo +nightly build -p vortex-ffi ## Usage from a CMake project -``` -# in vortex folder -cargo build --release -p vortex-ffi +CMake builds the Rust archive through Cargo; no separate `cargo build` is needed. Add the +repository root, or this directory alone, and link the static target: -# in your CMakeLists.txt -include_directory(vortex/vortex-ffi) -target_link_libraries(my_target, vortex_ffi_shared) -# or target_link_libraries(my_target, vortex_ffi) +```cmake +add_subdirectory(path/to/vortex vortex) +target_link_libraries(my_target PRIVATE Vortex::ffi_static) ``` +The target carries the headers in `cinclude/`, the archive, and the system libraries it needs. +Build options such as `VORTEX_CARGO_PROFILE`, `VORTEX_ENABLE_CUDA`, and `VORTEX_SANITIZER` are +documented in the [C++ README](../lang/cpp/README.md) and apply to both layers. + ## Running C examples ```sh -cmake -Bbuild -DBUILD_EXAMPLES=1 -cmake --build build -./build/examples/dtype -./build/examples/scan -./build/examples/scan_to_arrow -./build/examples/write_sample +cmake -S . -B build -DVORTEX_BUILD_EXAMPLES=ON +cmake --build build --parallel +./build/examples/write_sample sample.vortex +./build/examples/dtype 'sample.vortex' +./build/examples/scan 'sample.vortex' +./build/examples/scan_to_arrow 'sample.vortex' ``` ## Testing C part -Build the test library: - -```sh -cmake -Bbuild -DBUILD_TESTS=1 -cmake --build build -``` - -Run the tests: +The tests use Catch2, so a C++ compiler is required: ```sh -ctest --test-dir build -j $(nproc) +cmake -S . -B build -DVORTEX_BUILD_TESTING=ON +cmake --build build --parallel +ctest --test-dir build --output-on-failure ``` -You will need C++ compiler toolchain to run the tests since they use Catch2. - ## Testing Rust part with sanitizers -AddressSanitizer: +The Rust tests run under a sanitizer with nightly `cargo test`. Substitute the native target +triple, for example `x86_64-unknown-linux-gnu`: ```sh # inside vortex-ffi -RUSTFLAGS="-Z sanitizer=address" \ -cargo +nightly test -Zbuild-std \ - --no-default-features --target \ - -- --no-capture -``` - -MemorySanitizer: - -```sh -RUSTFLAGS="-Z sanitizer=memory -Cunsafe-allow-abi-mismatch=sanitizer" \ -cargo +nightly test -Zbuild-std \ - --no-default-features --target \ - -- --no-capture +RUSTFLAGS="-Zsanitizer=address -Cunsafe-allow-abi-mismatch=sanitizer" \ +cargo +nightly test -Zbuild-std --target --tests -- --no-capture ``` -ThreadSanitizer: - -```sh -TSAN_OPTIONS="suppressions=$HOME/vortex/vortex-ffi/tsan_suppressions.txt" \ -RUSTFLAGS="-Z sanitizer=thread -Cunsafe-allow-abi-mismatch=sanitizer" \ -cargo +nightly test -Zbuild-std \ - --no-default-features --target \ - -- --no-capture -``` +Use `-Zsanitizer=memory` for MemorySanitizer and `-Zsanitizer=thread` for ThreadSanitizer; the +latter needs `TSAN_OPTIONS="suppressions=$PWD/tsan_suppressions.txt"`. - `-Zbuild-std` is needed as memory and thread sanitizers report std errors otherwise. -- `--no-default-features` is needed as we use Mimalloc otherwise which interferes with sanitizers. - `allow-abi-mismatch` is safe because in our dependency graph only crates like `compiler_builtins` unset sanitization, and they do it on purpose. +- `--tests` skips doctests, which rustdoc builds without `RUSTFLAGS` and which would therefore + mismatch the sanitizer-built dependencies. - Make sure to use `cargo test` and not `cargo nextest` as nextest reports less leaks. - If you want stack trace symbolization, install `llvm-symbolizer`. ## Testing Rust and C with sanitizers -1. Build FFI library with external sanitizer runtime: - -```sh -RUSTFLAGS="-Zsanitizer=address -Zexternal-clangrt" \ -cargo +nightly build -Zbuild-std --target= \ - --no-default-features -p vortex-ffi -``` - -2. Build tests with target triple: +CMake instruments the Rust archive, its C dependencies, and the tests together. `VORTEX_SANITIZER` +takes a comma-separated list of `asan`, `lsan`, `ubsan`, and `tsan`. Each instruments the C and C++ +code, and all but `ubsan` also instrument the Rust code, since rustc has no UBSan. Rust +instrumentation uses rustup's `nightly` toolchain, which needs the `rust-src` component, +unless `RUSTUP_TOOLCHAIN` selects another, and the C and C++ side needs Clang: ```sh -cmake -Bbuild -DSANITIZER=asan -DTARGET_TRIPLE= +cmake -S . -B build \ + -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ + -DVORTEX_SANITIZER=asan,ubsan -DVORTEX_SANITIZE_RUST_STD=ON \ + -DVORTEX_BUILD_TESTING=ON +cmake --build build --parallel +./build/test/vortex_ffi_test 2>&1 | rustfilt -i- ``` -3. Run the tests (ctest doesn't output failures in detail): - -```sh -./build/test/vortex_ffi_test 2>& 1 | rustfilt -i- -``` +For ThreadSanitizer use `tsan` and point `TSAN_OPTIONS` at `tsan_suppressions.txt`. diff --git a/vortex-ffi/cmake/CargoBuild.cmake b/vortex-ffi/cmake/CargoBuild.cmake new file mode 100644 index 00000000000..36918989ebe --- /dev/null +++ b/vortex-ffi/cmake/CargoBuild.cmake @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +# Builds and stages the Rust FFI static library used by the Vortex C++ target. +# Configure.cmake invokes this internal script automatically during the build. + +cmake_minimum_required(VERSION 3.28) + +include("${CMAKE_CURRENT_LIST_DIR}/Helpers.cmake") + +# Configure.cmake passes these required values as `-D` arguments when the +# `vortex_ffi_cargo_build` target launches this script with `cmake -P`. +function(_vortex_require_build_inputs) + foreach(_required IN ITEMS + VORTEX_CARGO_EXECUTABLE + VORTEX_RUSTC_EXECUTABLE + VORTEX_RUST_TARGET + VORTEX_CARGO_TARGET_DIR + VORTEX_CARGO_PROFILE + VORTEX_FFI_PACKAGE + VORTEX_CARGO_FFI_ARCHIVE + VORTEX_CMAKE_FFI_ARCHIVE + VORTEX_RUSTFLAGS + VORTEX_CFLAGS + VORTEX_CXXFLAGS + VORTEX_C_COMPILER + VORTEX_CXX_COMPILER + VORTEX_AR + VORTEX_RANLIB) + if(NOT DEFINED ${_required} OR "${${_required}}" STREQUAL "") + message(FATAL_ERROR "CargoBuild.cmake requires ${_required}") + endif() + endforeach() +endfunction() + +# Format the Rust target for `cc` environment variable names. +function(_vortex_cc_target_env_key output) + string(REPLACE "-" "_" _target_key "${VORTEX_RUST_TARGET}") + string(TOLOWER "${_target_key}" _target_key) + set(${output} "${_target_key}" PARENT_SCOPE) +endfunction() + +# Assemble the Cargo command that builds the selected FFI package as +# a static library with the selected Cargo profile. +function(_vortex_make_cargo_command output) + set(_command + "${VORTEX_CARGO_EXECUTABLE}" + rustc + --locked + --package "${VORTEX_FFI_PACKAGE}" + --lib + --crate-type=staticlib + --target "${VORTEX_RUST_TARGET}" + --target-dir "${VORTEX_CARGO_TARGET_DIR}" + --profile "${VORTEX_CARGO_PROFILE}") + if(VORTEX_CARGO_BUILD_STD) + list(APPEND _command -Zbuild-std) + endif() + set(${output} "${_command}" PARENT_SCOPE) +endfunction() + +# Prepend the selected Rust and CUDA toolchain directories to the ambient PATH. +function(_vortex_build_tool_path output) + get_filename_component(_path "${VORTEX_RUSTC_EXECUTABLE}" DIRECTORY) + + if(VORTEX_NVCC_EXECUTABLE) + get_filename_component(_nvcc_bin_dir "${VORTEX_NVCC_EXECUTABLE}" DIRECTORY) + string(APPEND _path ":${_nvcc_bin_dir}") + endif() + + if("$ENV{PATH}" MATCHES ";") + message(FATAL_ERROR "The CMake-owned Cargo build does not support semicolons in PATH") + elseif(NOT "$ENV{PATH}" STREQUAL "") + string(APPEND _path ":$ENV{PATH}") + endif() + + set(${output} "${_path}" PARENT_SCOPE) +endfunction() + +# Assemble the target-specific Cargo environment from the selected tools and +# flags. +function(_vortex_make_cargo_environment target_key_lower output) + # Cargo separates CARGO_ENCODED_RUSTFLAGS arguments with ASCII unit separator, + # and the cc crate reads shell-quoted words with CC_SHELL_ESCAPED_FLAGS. + string(ASCII 31 _separator) + string(JOIN "${_separator}" _rustflags ${VORTEX_RUSTFLAGS}) + _vortex_encode_shell_arguments(_cflags ${VORTEX_CFLAGS}) + _vortex_encode_shell_arguments(_cxxflags ${VORTEX_CXXFLAGS}) + _vortex_build_tool_path(_cargo_path) + + set(_environment + "PATH=${_cargo_path}" + "RUSTC=${VORTEX_RUSTC_EXECUTABLE}" + "CC_SHELL_ESCAPED_FLAGS=1" + "CC_${target_key_lower}=${VORTEX_C_COMPILER}" + "CXX_${target_key_lower}=${VORTEX_CXX_COMPILER}" + "AR_${target_key_lower}=${VORTEX_AR}" + "RANLIB_${target_key_lower}=${VORTEX_RANLIB}" + "CFLAGS_${target_key_lower}=${_cflags}" + "CXXFLAGS_${target_key_lower}=${_cxxflags}" + "CARGO_ENCODED_RUSTFLAGS=${_rustflags}") + + # Keep the toolchain selection seen at configure time, even when the + # ambient environment differs at build time. + if(VORTEX_RUSTUP_TOOLCHAIN) + list(APPEND _environment "RUSTUP_TOOLCHAIN=${VORTEX_RUSTUP_TOOLCHAIN}") + else() + list(APPEND _environment --unset=RUSTUP_TOOLCHAIN) + endif() + + if(VORTEX_CUDA_ROOT) + list(APPEND _environment "CUDA_PATH=${VORTEX_CUDA_ROOT}") + endif() + + if(VORTEX_APPLE_DEPLOYMENT_TARGET) + list(APPEND _environment + "MACOSX_DEPLOYMENT_TARGET=${VORTEX_APPLE_DEPLOYMENT_TARGET}") + endif() + + set(${output} "${_environment}" PARENT_SCOPE) +endfunction() + +_vortex_require_build_inputs() +get_filename_component(_workspace_root "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE) +_vortex_cc_target_env_key(_target_key) +_vortex_make_cargo_command(_cargo_command) +_vortex_make_cargo_environment("${_target_key}" _cargo_environment) + +# Run Cargo with the CMake-selected tools and flags. Cargo remains responsible +# for dependency tracking and incremental freshness. +execute_process( + COMMAND "${CMAKE_COMMAND}" -E env + ${_cargo_environment} + ${_cargo_command} + WORKING_DIRECTORY "${_workspace_root}" + RESULT_VARIABLE _cargo_result) +if(NOT _cargo_result EQUAL 0) + message(FATAL_ERROR + "Cargo failed while building ${VORTEX_FFI_PACKAGE} (${_cargo_result})") +endif() + +# Catch profile, target, or crate-output changes that invalidate the archive +# path predicted by Configure.cmake. +if(NOT EXISTS "${VORTEX_CARGO_FFI_ARCHIVE}") + message(FATAL_ERROR + "Cargo completed successfully but did not produce the expected " + "static archive: ${VORTEX_CARGO_FFI_ARCHIVE}") +endif() + +# Stage the Cargo archive at CMake's stable artifact path. Preserve its +# timestamp when the contents are unchanged to avoid unnecessary relinks. +get_filename_component(_destination_dir "${VORTEX_CMAKE_FFI_ARCHIVE}" DIRECTORY) +file(MAKE_DIRECTORY "${_destination_dir}") +file(COPY_FILE + "${VORTEX_CARGO_FFI_ARCHIVE}" "${VORTEX_CMAKE_FFI_ARCHIVE}" + ONLY_IF_DIFFERENT) diff --git a/vortex-ffi/cmake/Configure.cmake b/vortex-ffi/cmake/Configure.cmake new file mode 100644 index 00000000000..7b74917cbdf --- /dev/null +++ b/vortex-ffi/cmake/Configure.cmake @@ -0,0 +1,343 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +# Configures the Rust FFI library used by the Vortex C++ target. This module +# defines the Cargo build and exposes its output to the CMake build. + +include_guard(GLOBAL) + +include("${CMAKE_CURRENT_LIST_DIR}/Helpers.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/RustToolchain.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/SystemDependencies.cmake") + +# Use an explicit Cargo profile override or map the single-config CMake build +# type to a profile and artifact directory. Empty and unknown build types use +# Cargo's development profile, with a warning for unknown ones; multi-config +# generators remain unsupported. +function(_vortex_resolve_cargo_profile configuration_output profile_output artifact_directory_output) + if(CMAKE_CONFIGURATION_TYPES) + message(FATAL_ERROR + "The initial Vortex CMake integration supports single-config " + "generators only; use Ninja with CMAKE_BUILD_TYPE=Debug, Release, " + "RelWithDebInfo, or MinSizeRel") + endif() + + # Normalize user input for comparisons and CMAKE__FLAGS_ lookups. + string(TOUPPER "${CMAKE_BUILD_TYPE}" _build_type) + + if(VORTEX_CARGO_PROFILE) + # An explicit user override takes precedence over the CMake build-type mapping. + _vortex_reject_semicolon("VORTEX_CARGO_PROFILE" "${VORTEX_CARGO_PROFILE}") + if(NOT VORTEX_CARGO_PROFILE MATCHES "^[A-Za-z0-9_-]+$") + message(FATAL_ERROR + "VORTEX_CARGO_PROFILE must contain only letters, numbers, " + "underscores, or hyphens; got '${VORTEX_CARGO_PROFILE}'") + endif() + if(VORTEX_CARGO_PROFILE STREQUAL "test" OR VORTEX_CARGO_PROFILE STREQUAL "bench") + message(FATAL_ERROR + "VORTEX_CARGO_PROFILE=${VORTEX_CARGO_PROFILE} is unsupported because " + "its artifact location is not stable") + endif() + + set(_cargo_profile "${VORTEX_CARGO_PROFILE}") + elseif(_build_type STREQUAL "DEBUG") + # Debug maps to Cargo's built-in development profile. + set(_cargo_profile "dev") + elseif(_build_type STREQUAL "RELEASE") + # Release maps to Cargo's built-in optimized profile. + set(_cargo_profile "release") + elseif(_build_type STREQUAL "RELWITHDEBINFO") + # RelWithDebInfo keeps release optimizations and full debug information. + set(_cargo_profile "release_debug") + elseif(_build_type STREQUAL "MINSIZEREL") + # MinSizeRel uses the release profile optimized for binary size. + set(_cargo_profile "release_size") + elseif(_build_type STREQUAL "") + # An empty build type compiles C++ without optimization, which matches dev. + set(_cargo_profile "dev") + else() + # Unknown build types fall back to Cargo's development profile with a warning. + message(WARNING + "Vortex has no Cargo profile mapping for " + "CMAKE_BUILD_TYPE='${CMAKE_BUILD_TYPE}'; using Cargo profile dev. " + "Set VORTEX_CARGO_PROFILE to override it") + set(_cargo_profile "dev") + endif() + + if(_cargo_profile STREQUAL "dev") + set(_artifact_directory "debug") + elseif(_cargo_profile STREQUAL "release") + set(_artifact_directory "release") + else() + set(_artifact_directory "${_cargo_profile}") + endif() + + set(${configuration_output} "${_build_type}" PARENT_SCOPE) + set(${profile_output} "${_cargo_profile}" PARENT_SCOPE) + set(${artifact_directory_output} "${_artifact_directory}" PARENT_SCOPE) +endfunction() + +# Select the Cargo package and static archive that provide Vortex FFI for a +# CPU-only or CUDA-enabled build. Return the include directories and optional +# CUDA tools. Fail if the workspace manifest, lockfile, or selected package +# manifest is missing, or CUDA is requested outside Linux. +function(_vortex_resolve_ffi_package + workspace_root + package_output + archive_name_output + include_dirs_output + nvcc_output + cuda_root_output) + set(_package "vortex-ffi") + set(_archive_name "libvortex_ffi.a") + set(_manifest "${workspace_root}/vortex-ffi/Cargo.toml") + set(_include_dirs "${workspace_root}/vortex-ffi/cinclude") + + # Shadow parent-scope values so CPU-only builds return empty CUDA outputs. + set(_nvcc "") + set(_cuda_root "") + + if(VORTEX_ENABLE_CUDA) + if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(FATAL_ERROR "VORTEX_ENABLE_CUDA is supported on Linux only") + endif() + find_package(CUDAToolkit REQUIRED) + set(_package "vortex-cuda-ffi") + set(_archive_name "libvortex_cuda_ffi.a") + set(_manifest "${workspace_root}/vortex-cuda/ffi/Cargo.toml") + list(APPEND _include_dirs "${workspace_root}/vortex-cuda/ffi/cinclude") + # CMake's FindCUDAToolkit module sets these after find_package succeeds. + set(_nvcc "${CUDAToolkit_NVCC_EXECUTABLE}") + set(_cuda_root "${CUDAToolkit_TARGET_DIR}") + endif() + + if(NOT EXISTS "${workspace_root}/Cargo.toml" OR + NOT EXISTS "${workspace_root}/Cargo.lock" OR + NOT EXISTS "${_manifest}") + message(FATAL_ERROR + "Vortex's CMake source build requires a complete workspace " + "checkout containing ${_package}") + endif() + + set(${package_output} "${_package}" PARENT_SCOPE) + set(${archive_name_output} "${_archive_name}" PARENT_SCOPE) + set(${include_dirs_output} "${_include_dirs}" PARENT_SCOPE) + set(${nvcc_output} "${_nvcc}" PARENT_SCOPE) + set(${cuda_root_output} "${_cuda_root}" PARENT_SCOPE) +endfunction() + +# Validate and configure the optional sanitizers. VORTEX_SANITIZER is a comma- +# or semicolon-separated list of asan, lsan, ubsan, and tsan. Each entry +# instruments the C and C++ code that clang compiles, including Cargo-built C +# dependencies, and all but ubsan also instrument the Rust code through rustc, +# which has no UBSan. Sanitizers require Clang and a Debug build, and rustc +# itself rejects the nightly-only Rust flags on stable. Standard-library +# instrumentation is optional. +function(_vortex_resolve_sanitizer + configuration + native_flag_output + rustflags_output + build_std_output) + string(REPLACE "," ";" _sanitizers "${VORTEX_SANITIZER}") + string(TOLOWER "${_sanitizers}" _sanitizers) + + set(_native "") + set(_rust "") + foreach(_sanitizer IN LISTS _sanitizers) + if(_sanitizer STREQUAL "asan") + list(APPEND _native address) + list(APPEND _rust address) + elseif(_sanitizer STREQUAL "lsan") + list(APPEND _native leak) + list(APPEND _rust leak) + elseif(_sanitizer STREQUAL "ubsan") + list(APPEND _native undefined) + elseif(_sanitizer STREQUAL "tsan") + list(APPEND _native thread) + list(APPEND _rust thread) + else() + message(FATAL_ERROR + "VORTEX_SANITIZER accepts asan, lsan, ubsan, and tsan; got '${_sanitizer}'") + endif() + endforeach() + + # Rebuilding std is meaningful only when Rust itself is instrumented. + if(VORTEX_SANITIZE_RUST_STD AND NOT _rust) + message(FATAL_ERROR + "VORTEX_SANITIZE_RUST_STD=ON requires a Rust sanitizer: asan, lsan, or tsan") + endif() + + set(_native_flag "") + set(_rustflags "") + if(_native) + # Use Clang so Rust and native code share one compatible sanitizer runtime. + if(NOT CMAKE_C_COMPILER_ID MATCHES "^(AppleClang|Clang)$" OR + NOT CMAKE_CXX_COMPILER_ID MATCHES "^(AppleClang|Clang)$") + message(FATAL_ERROR + "Vortex sanitizer builds require Clang or AppleClang for C and C++; " + "found ${CMAKE_C_COMPILER_ID} and ${CMAKE_CXX_COMPILER_ID}") + endif() + if(NOT configuration STREQUAL "DEBUG") + message(FATAL_ERROR "Vortex sanitizer builds require CMAKE_BUILD_TYPE=Debug") + endif() + + list(JOIN _native "," _native_joined) + set(_native_flag "-fsanitize=${_native_joined}") + endif() + if(_rust) + list(JOIN _rust "," _rust_joined) + list(APPEND _rustflags + -A warnings + -Cunsafe-allow-abi-mismatch=sanitizer + -C debuginfo=2 + -C opt-level=0 + # Use the sanitizer runtime linked by the final C++ target for both languages. + -Zexternal-clangrt + "-Zsanitizer=${_rust_joined}") + endif() + + set(${native_flag_output} "${_native_flag}" PARENT_SCOPE) + set(${rustflags_output} "${_rustflags}" PARENT_SCOPE) + set(${build_std_output} "${VORTEX_SANITIZE_RUST_STD}" PARENT_SCOPE) +endfunction() + +# Reconstruct CMake's effective C and C++ flags for Cargo build scripts, then +# append deployment-target, sanitizer, and PIC requirements. Return the C/C++ +# flag lists. +function(_vortex_native_flags + configuration + apple_deployment_target + sanitizer_flag + cflags_output + cxxflags_output) + set(_cmake_c_flags "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${configuration}}") + set(_cmake_cxx_flags "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${configuration}}") + _vortex_reject_semicolon("effective CMAKE_C_FLAGS" "${_cmake_c_flags}") + _vortex_reject_semicolon("effective CMAKE_CXX_FLAGS" "${_cmake_cxx_flags}") + separate_arguments(_cflags UNIX_COMMAND "${_cmake_c_flags}") + separate_arguments(_cxxflags UNIX_COMMAND "${_cmake_cxx_flags}") + + if(apple_deployment_target) + # Match Cargo-built native code to CMake's minimum macOS version. + list(APPEND _cflags "-mmacosx-version-min=${apple_deployment_target}") + list(APPEND _cxxflags "-mmacosx-version-min=${apple_deployment_target}") + endif() + + if(sanitizer_flag) + # Instrument Cargo-built native code with the selected sanitizer. + list(APPEND _cflags "${sanitizer_flag}") + list(APPEND _cxxflags "${sanitizer_flag}") + endif() + + # Native dependencies become part of the archive embedded in shared parents. + list(APPEND _cflags -fPIC) + list(APPEND _cxxflags -fPIC) + + set(${cflags_output} "${_cflags}" PARENT_SCOPE) + set(${cxxflags_output} "${_cxxflags}" PARENT_SCOPE) +endfunction() + +# Configure Cargo and expose the staged FFI archive as a private dependency of +# the public Vortex C++ target. +block(SCOPE_FOR VARIABLES) + _vortex_resolve_cargo_profile(_configuration _cargo_profile _cargo_artifact_directory) + + get_filename_component(_workspace_root "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE) + _vortex_resolve_ffi_package( + "${_workspace_root}" + _ffi_package + _cargo_archive_name + _ffi_include_dirs + _nvcc_executable + _cuda_root) + + _vortex_resolve_rust_toolchain("${_workspace_root}") + _vortex_resolve_sanitizer( + "${_configuration}" + _sanitizer_compile_flag + _sanitizer_rustflags + _cargo_build_std) + # Rust sanitizers need nightly-only rustc flags; default to rustup's + # nightly unless the environment already selected a toolchain. + if(_sanitizer_rustflags AND VORTEX_RUSTUP_TOOLCHAIN STREQUAL "") + set(VORTEX_RUSTUP_TOOLCHAIN nightly) + endif() + + if(NOT "$ENV{CARGO_ENCODED_RUSTFLAGS}" STREQUAL "" OR NOT "$ENV{RUSTFLAGS}" STREQUAL "") + message(STATUS "Vortex ignores ambient Rust flags in its Cargo build") + endif() + + _vortex_native_flags( + "${_configuration}" + "${VORTEX_APPLE_DEPLOYMENT_TARGET}" + "${_sanitizer_compile_flag}" + _native_c_flags + _native_cxx_flags) + # Frame pointers and PIC apply to every Rust build; the archive is embedded + # in shared parents. + set(_rustflags ${_sanitizer_rustflags} -C force-frame-pointers=yes -C relocation-model=pic) + + # Cargo owns incremental invalidation inside this CMake-build-local cache. + # Registering the directory as additional clean state gives the standard + # CMake clean target the same effect as `cargo clean --target-dir ...`. + set(_cargo_target_dir "${CMAKE_CURRENT_BINARY_DIR}/cargo-target") + set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_CLEAN_FILES "${_cargo_target_dir}") + set(_cargo_ffi_archive + "${_cargo_target_dir}/${VORTEX_RUST_TARGET}/${_cargo_artifact_directory}/${_cargo_archive_name}") + set(_ffi_archive "${CMAKE_CURRENT_BINARY_DIR}/vortex-artifacts/libvortex_ffi.a") + + # Each value below becomes one `-D` argument of the driver and later one + # environment entry, where a semicolon would split it. The flag variables + # are passed as lists on purpose. + foreach(_name IN ITEMS + VORTEX_CARGO_EXECUTABLE VORTEX_RUSTC_EXECUTABLE VORTEX_RUSTUP_TOOLCHAIN + VORTEX_APPLE_DEPLOYMENT_TARGET + CMAKE_C_COMPILER CMAKE_CXX_COMPILER CMAKE_AR CMAKE_RANLIB) + _vortex_reject_semicolon("${_name}" "${${_name}}") + endforeach() + + # The phony target lets Cargo own dependency tracking. Copy-if-different in + # the driver prevents fresh Cargo checks from forcing downstream relinks. + add_custom_target(vortex_ffi_cargo_build + COMMAND "${CMAKE_COMMAND}" + "-DVORTEX_CARGO_EXECUTABLE=${VORTEX_CARGO_EXECUTABLE}" + "-DVORTEX_RUSTC_EXECUTABLE=${VORTEX_RUSTC_EXECUTABLE}" + "-DVORTEX_RUSTUP_TOOLCHAIN=${VORTEX_RUSTUP_TOOLCHAIN}" + "-DVORTEX_RUST_TARGET=${VORTEX_RUST_TARGET}" + "-DVORTEX_CARGO_TARGET_DIR=${_cargo_target_dir}" + "-DVORTEX_CARGO_PROFILE=${_cargo_profile}" + "-DVORTEX_FFI_PACKAGE=${_ffi_package}" + "-DVORTEX_CARGO_FFI_ARCHIVE=${_cargo_ffi_archive}" + "-DVORTEX_NVCC_EXECUTABLE=${_nvcc_executable}" + "-DVORTEX_CUDA_ROOT=${_cuda_root}" + "-DVORTEX_CARGO_BUILD_STD=${_cargo_build_std}" + "-DVORTEX_CMAKE_FFI_ARCHIVE=${_ffi_archive}" + "-DVORTEX_RUSTFLAGS=${_rustflags}" + "-DVORTEX_CFLAGS=${_native_c_flags}" + "-DVORTEX_CXXFLAGS=${_native_cxx_flags}" + "-DVORTEX_C_COMPILER=${CMAKE_C_COMPILER}" + "-DVORTEX_CXX_COMPILER=${CMAKE_CXX_COMPILER}" + "-DVORTEX_AR=${CMAKE_AR}" + "-DVORTEX_RANLIB=${CMAKE_RANLIB}" + "-DVORTEX_APPLE_DEPLOYMENT_TARGET=${VORTEX_APPLE_DEPLOYMENT_TARGET}" + -P "${CMAKE_CURRENT_LIST_DIR}/CargoBuild.cmake" + BYPRODUCTS "${_ffi_archive}" + COMMENT "Building the PIC Vortex FFI static archive with Cargo" + USES_TERMINAL + VERBATIM) + + # Global so that sibling directories such as lang/cpp can link it. + add_library(vortex_ffi_static STATIC IMPORTED GLOBAL) + set_target_properties(vortex_ffi_static PROPERTIES + IMPORTED_LOCATION "${_ffi_archive}" + INTERFACE_INCLUDE_DIRECTORIES "${_ffi_include_dirs}") + add_dependencies(vortex_ffi_static vortex_ffi_cargo_build) + _vortex_attach_system_dependencies(vortex_ffi_static "${VORTEX_RUST_TARGET}") + if(_sanitizer_compile_flag) + target_compile_options(vortex_ffi_static INTERFACE "${_sanitizer_compile_flag}") + target_link_options(vortex_ffi_static INTERFACE "${_sanitizer_compile_flag}") + endif() + + message(STATUS "Vortex Rust target: ${VORTEX_RUST_TARGET}") + message(STATUS "Vortex Cargo target directory: ${_cargo_target_dir}") +endblock() diff --git a/vortex-ffi/cmake/Helpers.cmake b/vortex-ffi/cmake/Helpers.cmake new file mode 100644 index 00000000000..57739f9d2b3 --- /dev/null +++ b/vortex-ffi/cmake/Helpers.cmake @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +# Provides shared internal utilities for the Vortex CMake integration. + +include_guard(GLOBAL) + +# Fail if a scalar value contains CMake's semicolon list separator. +function(_vortex_reject_semicolon label value) + if("${value}" MATCHES ";") + message(FATAL_ERROR + "${label} contains a semicolon, which is unsupported because " + "CMake uses semicolons as list separators: ${value}") + endif() +endfunction() + +# Encode ARGN as POSIX shell words in one space-delimited string. +function(_vortex_encode_shell_arguments output) + set(_encoded "") + foreach(_argument IN LISTS ARGN) + string(REPLACE "'" "'\"'\"'" _argument_quoted "${_argument}") + if(_encoded) + string(APPEND _encoded " ") + endif() + string(APPEND _encoded "'${_argument_quoted}'") + endforeach() + set(${output} "${_encoded}" PARENT_SCOPE) +endfunction() diff --git a/vortex-ffi/cmake/RustToolchain.cmake b/vortex-ffi/cmake/RustToolchain.cmake new file mode 100644 index 00000000000..7422855c554 --- /dev/null +++ b/vortex-ffi/cmake/RustToolchain.cmake @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +# Selects the Rust tools and the native Rust target for the Vortex C++ build. + +include_guard(GLOBAL) + +# Sets VORTEX_RUST_TARGET, VORTEX_RUSTUP_TOOLCHAIN, and +# VORTEX_APPLE_DEPLOYMENT_TARGET in the caller's scope. The cache entries +# VORTEX_CARGO_EXECUTABLE and VORTEX_RUSTC_EXECUTABLE hold the selected tools. +function(_vortex_resolve_rust_toolchain workspace_root) + # Rustup proxies honor the workspace toolchain file when run from the + # workspace. RUSTUP_TOOLCHAIN is captured so Cargo builds keep this selection. + find_program(VORTEX_CARGO_EXECUTABLE NAMES cargo REQUIRED) + find_program(VORTEX_RUSTC_EXECUTABLE NAMES rustc REQUIRED) + execute_process( + COMMAND "${VORTEX_RUSTC_EXECUTABLE}" -vV + WORKING_DIRECTORY "${workspace_root}" + OUTPUT_VARIABLE _rustc_verbose + COMMAND_ERROR_IS_FATAL ANY) + string(REGEX MATCH "host: ([^\r\n]+)" _match "${_rustc_verbose}") + set(VORTEX_RUST_TARGET "${CMAKE_MATCH_1}" PARENT_SCOPE) + set(VORTEX_RUSTUP_TOOLCHAIN "$ENV{RUSTUP_TOOLCHAIN}" PARENT_SCOPE) + + # Without an explicit deployment target the cc crate uses the SDK version, + # which can exceed CMake's link target and makes ld64 warn about every + # Cargo-built C object; 11.0 is rustc's minimum for aarch64-apple-darwin. + if(APPLE AND NOT CMAKE_OSX_DEPLOYMENT_TARGET) + set(VORTEX_APPLE_DEPLOYMENT_TARGET "11.0" PARENT_SCOPE) + else() + set(VORTEX_APPLE_DEPLOYMENT_TARGET "${CMAKE_OSX_DEPLOYMENT_TARGET}" PARENT_SCOPE) + endif() +endfunction() diff --git a/vortex-ffi/cmake/SystemDependencies.cmake b/vortex-ffi/cmake/SystemDependencies.cmake new file mode 100644 index 00000000000..d52c81a9697 --- /dev/null +++ b/vortex-ffi/cmake/SystemDependencies.cmake @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +# Defines the system libraries required to link the Rust FFI archive into C++ +# targets. + +include_guard(GLOBAL) + +# Attach the platform libraries required by the Rust static archive; fail if the +# Rust target has no supported link manifest. +function(_vortex_attach_system_dependencies target rust_target) + if(rust_target MATCHES "^(x86_64|aarch64)-unknown-linux-gnu$") + if(NOT TARGET Threads::Threads) + set(THREADS_PREFER_PTHREAD_FLAG TRUE) + find_package(Threads REQUIRED) + endif() + + target_link_libraries("${target}" INTERFACE + gcc_s + util + rt + Threads::Threads + m + ${CMAKE_DL_LIBS} + c) + elseif(rust_target STREQUAL "aarch64-apple-darwin") + # The archive needs no C++ runtime, so C and C++ consumers alike only + # need libSystem from their driver plus these extra libraries. + find_library(_vortex_core_foundation CoreFoundation REQUIRED NO_CACHE) + target_link_libraries("${target}" INTERFACE iconv "${_vortex_core_foundation}") + else() + message(FATAL_ERROR + "Vortex has no validated native static-link manifest for Rust target " + "${rust_target}") + endif() +endfunction() diff --git a/vortex-ffi/examples/CMakeLists.txt b/vortex-ffi/examples/CMakeLists.txt index 47228d9a903..4763a0b0513 100644 --- a/vortex-ffi/examples/CMakeLists.txt +++ b/vortex-ffi/examples/CMakeLists.txt @@ -1,18 +1,16 @@ # SPDX-License-Identifier: CC-BY-4.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -# allow linking with vortex_ffi_shared although it's not in current folder -cmake_policy(SET CMP0079 NEW) - -add_executable(scan scan.c) -target_link_libraries(scan PRIVATE vortex_ffi_shared) - -add_executable(scan_to_arrow scan_to_arrow.c) -target_link_libraries(scan_to_arrow PRIVATE - nanoarrow_shared vortex_ffi_shared) - -add_executable(dtype dtype.c) -target_link_libraries(dtype PRIVATE vortex_ffi_shared) - -add_executable(write_sample write_sample.c) -target_link_libraries(write_sample PRIVATE vortex_ffi_shared) +# Target names carry a prefix so a build that also includes the C++ examples +# has no clashes; the executables keep their plain names. +foreach(_name scan scan_to_arrow dtype write_sample) + add_executable(ffi_${_name} ${_name}.c) + set_target_properties(ffi_${_name} PROPERTIES OUTPUT_NAME ${_name}) + target_compile_features(ffi_${_name} PRIVATE c_std_17) + target_compile_options(ffi_${_name} PRIVATE -Wall -Wextra -Wpedantic) + if(VORTEX_WARNINGS_AS_ERRORS) + target_compile_options(ffi_${_name} PRIVATE -Werror) + endif() + target_link_libraries(ffi_${_name} PRIVATE Vortex::ffi_static) +endforeach() +target_link_libraries(ffi_scan_to_arrow PRIVATE nanoarrow_shared) diff --git a/vortex-ffi/examples/scan.c b/vortex-ffi/examples/scan.c index 67df597c69f..b91c8ec1c5e 100644 --- a/vortex-ffi/examples/scan.c +++ b/vortex-ffi/examples/scan.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: CC-BY-4.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors #include "vortex.h" +#include #include #include #include @@ -16,10 +17,10 @@ void print_estimate(const char *what, const vx_estimate *estimate) { printf("%s: unknown\n", what); return; case VX_ESTIMATE_EXACT: - printf("%s: %lu\n", what, estimate->estimate); + printf("%s: %" PRIu64 "\n", what, estimate->estimate); return; case VX_ESTIMATE_INEXACT: - printf("%s: approximately %lu\n", what, estimate->estimate); + printf("%s: approximately %" PRIu64 "\n", what, estimate->estimate); break; } } @@ -30,7 +31,7 @@ void print_error(const char *what, const vx_error *error) { } struct scan_thread_info { - pthread_t thread_id; + size_t thread_id; pthread_mutex_t *mutex; vx_scan *scan; size_t partitions, arrays, rows; @@ -59,7 +60,7 @@ void *execute_scan_thread(void *arg) { return NULL; } - printf("Thread %lu processing partition %lu, ", info->thread_id + 1, info->partitions); + printf("Thread %zu processing partition %zu, ", info->thread_id + 1, info->partitions); print_estimate("row count", &row_count); // An array is a batch of rows from a partition @@ -77,21 +78,18 @@ void *execute_scan_thread(void *arg) { } } - printf("Thread %lu finished, processed %lu partitions, %lu arrays, %lu rows\n", - info->thread_id + 1, - info->partitions, - info->arrays, - info->rows); + printf("Thread %zu finished, processed %zu partitions, %zu arrays, %zu rows\n", info->thread_id + 1, + info->partitions, info->arrays, info->rows); return NULL; } -vx_error *execute_scan(vx_scan *scan, pthread_t num_threads) { +vx_error *execute_scan(vx_scan *scan, size_t num_threads) { pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; pthread_t threads[MAX_THREADS]; struct scan_thread_info infos[MAX_THREADS] = {0}; - printf("Starting scan, using %lu threads\n", num_threads); - for (pthread_t id = 0; id < num_threads; ++id) { + printf("Starting scan, using %zu threads\n", num_threads); + for (size_t id = 0; id < num_threads; ++id) { struct scan_thread_info *info = &infos[id]; info->thread_id = id; info->mutex = &mutex; @@ -100,7 +98,7 @@ vx_error *execute_scan(vx_scan *scan, pthread_t num_threads) { } size_t partitions = 0, arrays = 0, rows = 0; - for (pthread_t id = 0; id < num_threads; ++id) { + for (size_t id = 0; id < num_threads; ++id) { pthread_join(threads[id], NULL); struct scan_thread_info *info = &infos[id]; @@ -114,11 +112,11 @@ vx_error *execute_scan(vx_scan *scan, pthread_t num_threads) { rows += info->rows; } - printf("Finished scan, processed %lu partitions, %lu arrays, %lu rows\n", partitions, arrays, rows); + printf("Finished scan, processed %zu partitions, %zu arrays, %zu rows\n", partitions, arrays, rows); return NULL; } -int parse_options(int argc, char *argv[], pthread_t *threads, char **paths) { +int parse_options(int argc, char *argv[], size_t *threads, char **paths) { int opt; while ((opt = getopt(argc, argv, "j:")) != -1) { switch (opt) { @@ -132,7 +130,7 @@ int parse_options(int argc, char *argv[], pthread_t *threads, char **paths) { } if (*threads != 0 && (*threads < 1 || *threads > MAX_THREADS)) { - fprintf(stderr, "Invalid thread count %lu, expected [1; 64]\n", *threads); + fprintf(stderr, "Invalid thread count %zu, expected [1; 64]\n", *threads); return 1; } @@ -146,7 +144,7 @@ int parse_options(int argc, char *argv[], pthread_t *threads, char **paths) { } int main(int argc, char *argv[]) { - pthread_t threads = 0; + size_t threads = 0; char *paths; if (parse_options(argc, argv, &threads, &paths)) { return 1; diff --git a/vortex-ffi/test/CMakeLists.txt b/vortex-ffi/test/CMakeLists.txt index be0288ef954..10499b59a10 100644 --- a/vortex-ffi/test/CMakeLists.txt +++ b/vortex-ffi/test/CMakeLists.txt @@ -1,21 +1,34 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -include(CTest) + FetchContent_Declare( - Catch - GIT_REPOSITORY https://github.com/catchorg/Catch2.git - GIT_TAG v3.8.1 -) + Catch + SYSTEM + GIT_REPOSITORY https://github.com/catchorg/Catch2.git + GIT_TAG v3.8.1) FetchContent_MakeAvailable(Catch) -include(Catch) +# By path: Catch2 exports its module directory only to the directory that +# populated it, which may be a parent that fetched Catch2 first. +include("${catch_SOURCE_DIR}/extras/Catch.cmake") + +# Catch2 enables std::string_view, optional, variant, and byte support from +# C++17, in headers and compiled sources alike; build it with those on so it +# matches the C++20 tests that include its headers. +target_compile_features(Catch2 PRIVATE cxx_std_17) # https://github.com/catchorg/Catch2/issues/1833 target_compile_definitions(Catch2 PRIVATE CATCH_CONFIG_NO_POSIX_SIGNALS) -file(GLOB TEST_FILES "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") -message(NOTICE "Test files ${TEST_FILES}") -add_executable(vortex_ffi_test ${TEST_FILES}) +file(GLOB _vortex_ffi_test_sources CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") + +add_executable(vortex_ffi_test ${_vortex_ffi_test_sources}) +target_compile_features(vortex_ffi_test PRIVATE cxx_std_20) +target_compile_options(vortex_ffi_test PRIVATE -Wall -Wextra -Wpedantic) +if(VORTEX_WARNINGS_AS_ERRORS) + target_compile_options(vortex_ffi_test PRIVATE -Werror) +endif() target_link_libraries(vortex_ffi_test PRIVATE - vortex_ffi_shared + Vortex::ffi_static Catch2::Catch2WithMain nanoarrow_shared) + catch_discover_tests(vortex_ffi_test)