New C++ API tests

.github/workflows/ci.yml

@@ -211,7 +211,7 @@ jobs:
           # including all private docs.
           cargo doc --profile ci --no-deps --document-private-items --workspace --exclude vortex-python --exclude vortex-python-cuda
           # nextest doesn't support doc tests, so we run it here
-          cargo test --profile ci --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --exclude vortex-python-cuda --no-fail-fast
+          cargo test --profile ci --doc --workspace --all-features --exclude vortex-jni --exclude vortex-ffi --exclude xtask --exclude vortex-python-cuda --no-fail-fast
 
   build-rust:
     name: "Rust build (${{matrix.config.name}})"
@@ -553,8 +553,11 @@ jobs:
       - name: Build C++ library (asan)
         run: |
           mkdir -p build
-          cmake -S lang/cpp -B build -DSANITIZER=asan -DTARGET_TRIPLE="x86_64-unknown-linux-gnu"
+          cmake -S lang/cpp -Bbuild -DSANITIZER=asan -DBUILD_TESTS=1 -DTARGET_TRIPLE="x86_64-unknown-linux-gnu"
           cmake --build build --parallel $(nproc)
+      - name: Run C++ tests
+        run: |
+          build/tests/vortex_cxx_test
 
   sqllogic-test:
     name: "SQL logic tests"

.github/workflows/rust-instrumented.yml

@@ -101,7 +101,7 @@ jobs:
             --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \
             --ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \
             --ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \
-            --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \
+            --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' \
             --ignore benchmarks/* --ignore 'vortex-test/*' \
             -o ${{ env.GRCOV_OUTPUT_FILE }}
           test -s ${{ env.GRCOV_OUTPUT_FILE }}
@@ -114,6 +114,42 @@ jobs:
           flags: ${{ matrix.suite }}
           use_oidc: true
 
+  cxx-coverage:
+    name: "C++ API (coverage)"
+    timeout-minutes: 15
+    permissions:
+      id-token: write
+    runs-on: >-
+      ${{ github.repository == 'vortex-data/vortex'
+          && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-coverage', github.run_id)
+          || 'ubuntu-latest' }}
+    steps:
+      - uses: runs-on/action@v2
+        if: github.repository == 'vortex-data/vortex'
+        with:
+          sccache: s3
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7
+      - uses: ./.github/actions/setup-prebuild
+        with:
+          enable-sccache: "true"
+      - name: Install lcov
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y lcov libjson-xs-perl
+      - name: Build FFI library
+        run: cargo build -p vortex-ffi
+      - name: Build and test C++ API with coverage
+        working-directory: lang/cpp
+        run: ./gcov-report.sh
+      - name: Codecov
+        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f  # v7
+        with:
+          name: cxx-api
+          files: lang/cpp/coverage.info
+          disable_search: true
+          flags: cxx-api
+          use_oidc: true
+
   rust-test-sanitizer:
     strategy:
       fail-fast: false

docs/api/cpp/index.rst

@@ -164,8 +164,8 @@ Source code for this example is `writer.cpp
 
 .. code-block:: cpp
 
-    const Session session;
-    const DataType dtype = dtype::struct_({
+    Session session;
+    DataType dtype = dtype::struct_({
         {"age", dtype::uint8()},
         {"height", dtype::uint16(Nullable)},
     });
@@ -187,15 +187,14 @@ Source code for this example is `writer.cpp
     Expression age_gt_10 = expr::gt(expr::col("age"), expr::lit<uint8_t>(10));
     Array validity_array = array.apply(age_gt_10);
 
-    const Validity validity = Validity::from_array(validity_array);
+    Validity validity = Validity::from_array(validity_array);
     Array array2 = make_struct({
         {"age", age},
         {"height", Array::primitive<uint16_t>(height_buffer, validity)},
     });
 
     Writer writer = Writer::open(session, argv[1], dtype);
-    writer.push(array);
-    writer.push(array2);
+    writer.push({array, array2});
     writer.finish();
 
 DataType

lang/cpp/.gitignore

@@ -0,0 +1 @@
+coverage*

lang/cpp/CMakeLists.txt

@@ -21,6 +21,8 @@ else ()
     message(STATUS "Sccache not found")
 endif ()
 
+option(BUILD_TESTS "Build tests" OFF)
+
 set(SANITIZER "" CACHE STRING "Build with sanitizers")
 set(TARGET_TRIPLE "" CACHE STRING "Rust target triple for FFI library")
 set(RUST_BUILD_PROFILE "" CACHE STRING "Cargo profile name for Rust FFI library")
@@ -122,3 +124,24 @@ if(TARGET vortex_ffi_shared)
         target_link_libraries(vortex_cxx_shared PRIVATE ${APPLE_LINK_FLAGS})
     endif()
 endif()
+
+if (BUILD_TESTS)
+    FetchContent_Declare(
+      Nanoarrow
+      GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow
+      GIT_TAG        apache-arrow-nanoarrow-0.8.0
+    )
+    FetchContent_MakeAvailable(Nanoarrow)
+
+    FetchContent_Declare(
+        Catch
+        GIT_REPOSITORY https://github.com/catchorg/Catch2.git
+        GIT_TAG        v3.8.1
+    )
+    FetchContent_MakeAvailable(Catch)
+    include(Catch)
+    target_compile_definitions(Catch2 PRIVATE CATCH_CONFIG_NO_POSIX_SIGNALS)
+
+    include(CTest)
+    add_subdirectory(tests)
+endif()

lang/cpp/README.md

@@ -35,3 +35,11 @@ cmake -Bbuild -DBUILD_EXAMPLES=ON
 cmake --build build -j
 ./build/examples/hello-vortex
 ```
+
+## Check coverage
+
+This will generate an LCOV directory `coverage`:
+
+```sh
+./gcov-report.sh generate
+```

lang/cpp/gcov-report.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+set -eu
+cmake -Bbuild -DBUILD_TESTS=1 -DCMAKE_CXX_FLAGS='-fprofile-arcs -ftest-coverage'
+cmake --build build -j
+ctest --test-dir build --output-on-failure
+
+geninfo build/CMakeFiles/vortex_cxx_shared.dir/ \
+    build/tests/CMakeFiles/vortex_cxx_test.dir/ \
+    --rc geninfo_unexecuted_blocks=1 \
+    --exclude /usr --exclude build/_deps --exclude tests \
+    -j -b src -o coverage.info
+if [ $# -gt 0 ]; then
+    genhtml coverage.info -o coverage
+fi

lang/cpp/include/vortex/array.hpp

@@ -40,8 +40,8 @@ enum class ValidityType {
     AllValid = VX_VALIDITY_ALL_VALID,
     // All items are invalid
     AllInvalid = VX_VALIDITY_ALL_INVALID,
-    // Item validity is set in a boolean array: true = valid, false = invalid
-    Array = VX_VALIDITY_ARRAY,
+    // Item validity is set from a boolean array: true = valid, false = invalid
+    FromArray = VX_VALIDITY_ARRAY,
 };
 
 /**

lang/cpp/include/vortex/common.hpp

@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 #pragma once
 
+#include <bit>
 #include <cstdint>
 #include <span>
 #include <string_view>
@@ -13,9 +14,34 @@
 namespace vortex {
 struct float16_t {
     uint16_t bits;
-    friend bool operator==(float16_t, float16_t) = default;
+    constexpr friend bool operator==(float16_t, float16_t) = default;
     // NOLINTNEXTLINE
-    operator float() const;
+    constexpr operator float() const {
+        float result;
+        const uint32_t sign = (bits >> 15) & 1;
+        const uint32_t exponent = (bits >> 10) & 0x1F;
+        const uint32_t mantissa = bits & 0x3FF;
+
+        uint32_t out;
+        if (exponent == 0x1F) {
+            out = (sign << 31) | 0x7F800000 | (mantissa << 13);
+        } else if (exponent == 0) {
+            if (mantissa == 0) {
+                out = sign << 31;
+            } else {
+                uint32_t m = mantissa;
+                int e = -1;
+                do {
+                    m <<= 1;
+                    ++e;
+                } while ((m & 0x400) == 0);
+                out = (sign << 31) | ((127 - 15 - e) << 23) | ((m & 0x3FF) << 13);
+            }
+        } else {
+            out = (sign << 31) | ((exponent - 15 + 127) << 23) | (mantissa << 13);
+        }
+        return std::bit_cast<float>(out);
+    }
 };
 static_assert(sizeof(float16_t) == 2 && std::is_trivially_copyable_v<float16_t>);
 } // namespace vortex
@@ -64,6 +90,10 @@ constexpr vx_ptype to_ptype() {
         return PTYPE_I32;
     } else if constexpr (std::is_same_v<T, int64_t>) {
         return PTYPE_I64;
+#if __STDCPP_FLOAT16_T__ == 1
+    } else if constexpr (std::is_same_v<T, _Float16>) {
+        return PTYPE_F16;
+#endif
     } else if constexpr (std::is_same_v<T, float16_t>) {
         return PTYPE_F16;
     } else if constexpr (std::is_same_v<T, float>) {
@@ -79,7 +109,13 @@ inline constexpr bool is_numeric_element =
     std::is_same_v<T, uint8_t> || std::is_same_v<T, uint16_t> || std::is_same_v<T, uint32_t> ||
     std::is_same_v<T, uint64_t> || std::is_same_v<T, int8_t> || std::is_same_v<T, int16_t> ||
     std::is_same_v<T, int32_t> || std::is_same_v<T, int64_t> || std::is_same_v<T, float> ||
-    std::is_same_v<T, double> || std::is_same_v<T, float16_t>;
+    std::is_same_v<T, double>
+#if __STDCPP_FLOAT16_T__ == 1
+    || std::is_same_v<T, _Float16>
+#else
+    || std::is_same_v<T, float16_t>
+#endif
+    ;
 } // namespace vortex::detail
 
 namespace vortex {

lang/cpp/include/vortex/scan.hpp

@@ -20,8 +20,9 @@
 namespace vortex {
 
 namespace detail {
+
 // range-for support for Scan and Partition
-template <class Source, class Item, auto Next>
+template <class Source, class Item, std::optional<Item> (Source::*Next)()>
 class PullRange {
 public:
     class iterator {
@@ -37,7 +38,7 @@ class PullRange {
             return *cur_;
         }
         iterator &operator++() {
-            cur_ = Next(src_);
+            cur_ = (src_->*Next)();
             return *this;
         }
         void operator++(int) {
@@ -55,7 +56,7 @@ class PullRange {
     explicit PullRange(Source &src) : src_(&src) {
     }
     iterator begin() {
-        return iterator(src_, Next(src_));
+        return iterator(src_, (src_->*Next)());
     }
     std::default_sentinel_t end() {
         return std::default_sentinel;

lang/cpp/include/vortex/writer.hpp

@@ -18,6 +18,8 @@ namespace vortex {
  *
  * finish() writes the footer and finalizes the file.
  * Not calling finish() leaves file corrupted.
+ *
+ * Writer methods are thread-unsafe.
  */
 class Writer {
 public:
@@ -32,7 +34,9 @@ class Writer {
      * Append Array to output file.
      * Throws if "array"'s DataType doesn't match writer's DataType.
      */
+    void push(std::span<const Array> arrays);
     void push(const Array &array);
+    void push(std::initializer_list<Array> arrays);
 
     /*
      * Write footer and finalize the file.

lang/cpp/src/array.cpp

@@ -23,14 +23,17 @@ Validity::Validity(const Validity &other)
 }
 
 Validity::Validity(ValidityType type) : type_(type), array_(nullptr) {
-    if (type == ValidityType::Array) {
-        throw VortexException("Validity(ValidityType) called with ValidityType::Array",
+    if (type == ValidityType::FromArray) {
+        throw VortexException("Validity(ValidityType) called with ValidityType::FromArray",
                               ErrorCode::InvalidArgument);
     }
 }
 
 Validity Validity::from_array(const Array &bools) {
-    return {ValidityType::Array, vx_array_clone(Access::c_ptr(bools))};
+    if (!bools.has_dtype(DataTypeVariant::Bool)) {
+        throw VortexException("Validity array isn't a Bool array", ErrorCode::InvalidArgument);
+    }
+    return {ValidityType::FromArray, vx_array_clone(Access::c_ptr(bools))};
 }
 
 Validity::Validity(Validity &&other) noexcept : type_(other.type_), array_(other.array_) {
@@ -61,7 +64,7 @@ Validity::~Validity() {
 }
 
 Array Validity::array() const {
-    if (type_ != ValidityType::Array || array_ == nullptr) {
+    if (type_ != ValidityType::FromArray || array_ == nullptr) {
         throw VortexException("validity has no backing array", ErrorCode::InvalidArgument);
     }
     return Access::adopt<Array>(vx_array_clone(array_));
@@ -93,7 +96,7 @@ ValidityBits::ValidityBits(const Session &session, const vx_array *canonical) {
     case ValidityType::AllInvalid:
         all_invalid_ = true;
         return;
-    case ValidityType::Array:
+    case ValidityType::FromArray:
         break;
     }
 
@@ -163,7 +166,7 @@ Array Array::primitive_raw(vx_ptype ptype, const void *data, size_t len, const V
     std::optional<Array> keep_alive;
     vx_validity raw {};
     raw.type = static_cast<vx_validity_type>(validity.type());
-    if (validity.type() == ValidityType::Array) {
+    if (validity.type() == ValidityType::FromArray) {
         keep_alive = validity.array();
         raw.array = Access::c_ptr(*keep_alive);
     }
@@ -264,7 +267,7 @@ Array make_struct(std::span<const ColumnField> fields, const Validity &validity)
     raw.type = static_cast<vx_validity_type>(validity.type());
 
     std::optional<Array> keep_alive;
-    if (validity.type() == ValidityType::Array) {
+    if (validity.type() == ValidityType::FromArray) {
         keep_alive = validity.array();
         raw.array = Access::c_ptr(*keep_alive);
     }