From 05b5c8b9f144f8b981ae315e1dc7fc128d0a816e Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Tue, 14 Jul 2026 11:44:40 +0100 Subject: [PATCH 1/3] C++ api tests Signed-off-by: Mikhail Kot --- .github/workflows/ci.yml | 5 +- .github/workflows/rust-instrumented.yml | 36 +++ lang/cpp/.gitignore | 1 + lang/cpp/CMakeLists.txt | 23 ++ lang/cpp/README.md | 8 + lang/cpp/gcov-report.sh | 15 + lang/cpp/include/vortex/array.hpp | 4 +- lang/cpp/include/vortex/common.hpp | 30 +- lang/cpp/include/vortex/scan.hpp | 7 +- lang/cpp/src/array.cpp | 12 +- lang/cpp/src/f16.cpp | 40 --- lang/cpp/tests/CMakeLists.txt | 24 ++ lang/cpp/tests/array.cpp | 175 ++++++++++++ lang/cpp/tests/arrow.cpp | 115 ++++++++ lang/cpp/tests/common.hpp | 110 ++++++++ lang/cpp/tests/dtype.cpp | 81 ++++++ lang/cpp/tests/enum_sizes.cpp | 15 + lang/cpp/tests/expression.cpp | 102 +++++++ lang/cpp/tests/float16_t.cpp | 24 ++ lang/cpp/tests/float16_t_compat.cpp | 34 +++ lang/cpp/tests/scalar.cpp | 100 +++++++ lang/cpp/tests/scan.cpp | 348 ++++++++++++++++++++++++ lang/cpp/tests/session.cpp | 16 ++ lang/cpp/tests/string_binary.cpp | 133 +++++++++ lang/cpp/tests/writer.cpp | 53 ++++ 25 files changed, 1457 insertions(+), 54 deletions(-) create mode 100644 lang/cpp/.gitignore create mode 100755 lang/cpp/gcov-report.sh delete mode 100644 lang/cpp/src/f16.cpp create mode 100644 lang/cpp/tests/CMakeLists.txt create mode 100644 lang/cpp/tests/array.cpp create mode 100644 lang/cpp/tests/arrow.cpp create mode 100644 lang/cpp/tests/common.hpp create mode 100644 lang/cpp/tests/dtype.cpp create mode 100644 lang/cpp/tests/enum_sizes.cpp create mode 100644 lang/cpp/tests/expression.cpp create mode 100644 lang/cpp/tests/float16_t.cpp create mode 100644 lang/cpp/tests/float16_t_compat.cpp create mode 100644 lang/cpp/tests/scalar.cpp create mode 100644 lang/cpp/tests/scan.cpp create mode 100644 lang/cpp/tests/session.cpp create mode 100644 lang/cpp/tests/string_binary.cpp create mode 100644 lang/cpp/tests/writer.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7aae5dc647..bc41ec4d2bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -211,7 +211,7 @@ jobs: # including all private docs. cargo doc --profile ci --no-deps --document-private-items --workspace --exclude vortex-python --exclude vortex-python-cuda # nextest doesn't support doc tests, so we run it here - cargo test --profile ci --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --exclude vortex-python-cuda --no-fail-fast + cargo test --profile ci --doc --workspace --all-features --exclude vortex-jni --exclude vortex-ffi --exclude xtask --exclude vortex-python-cuda --no-fail-fast build-rust: name: "Rust build (${{matrix.config.name}})" @@ -555,6 +555,9 @@ jobs: mkdir -p build cmake -S lang/cpp -B build -DSANITIZER=asan -DTARGET_TRIPLE="x86_64-unknown-linux-gnu" cmake --build build --parallel $(nproc) + - name: Run C++ tests + run: | + lang/cpp/build/tests/vortex_cxx_test sqllogic-test: name: "SQL logic tests" diff --git a/.github/workflows/rust-instrumented.yml b/.github/workflows/rust-instrumented.yml index 5430f1512fc..075b437588a 100644 --- a/.github/workflows/rust-instrumented.yml +++ b/.github/workflows/rust-instrumented.yml @@ -114,6 +114,42 @@ jobs: flags: ${{ matrix.suite }} use_oidc: true + cxx-coverage: + name: "C++ API (coverage)" + timeout-minutes: 15 + permissions: + id-token: write + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-coverage', github.run_id) + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: ./.github/actions/setup-prebuild + with: + enable-sccache: "true" + - name: Install lcov + run: | + sudo apt-get update + sudo apt-get install -y lcov + - name: Build FFI library + run: cargo build -p vortex-ffi + - name: Build and test C++ API with coverage + working-directory: lang/cpp + run: ./gcov-report.sh + - name: Codecov + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7 + with: + name: cxx-api + files: lang/cpp/coverage.info + disable_search: true + flags: cxx-api + use_oidc: true + rust-test-sanitizer: strategy: fail-fast: false diff --git a/lang/cpp/.gitignore b/lang/cpp/.gitignore new file mode 100644 index 00000000000..3ea99d6c913 --- /dev/null +++ b/lang/cpp/.gitignore @@ -0,0 +1 @@ +coverage* diff --git a/lang/cpp/CMakeLists.txt b/lang/cpp/CMakeLists.txt index 28099a44baa..8949c1977cc 100644 --- a/lang/cpp/CMakeLists.txt +++ b/lang/cpp/CMakeLists.txt @@ -21,6 +21,8 @@ else () message(STATUS "Sccache not found") endif () +option(BUILD_TESTS "Build tests" OFF) + set(SANITIZER "" CACHE STRING "Build with sanitizers") set(TARGET_TRIPLE "" CACHE STRING "Rust target triple for FFI library") set(RUST_BUILD_PROFILE "" CACHE STRING "Cargo profile name for Rust FFI library") @@ -122,3 +124,24 @@ if(TARGET vortex_ffi_shared) target_link_libraries(vortex_cxx_shared PRIVATE ${APPLE_LINK_FLAGS}) endif() endif() + +if (BUILD_TESTS) + FetchContent_Declare( + Nanoarrow + GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow + GIT_TAG apache-arrow-nanoarrow-0.8.0 + ) + FetchContent_MakeAvailable(Nanoarrow) + + FetchContent_Declare( + Catch + GIT_REPOSITORY https://github.com/catchorg/Catch2.git + GIT_TAG v3.8.1 + ) + FetchContent_MakeAvailable(Catch) + include(Catch) + target_compile_definitions(Catch2 PRIVATE CATCH_CONFIG_NO_POSIX_SIGNALS) + + include(CTest) + add_subdirectory(tests) +endif() diff --git a/lang/cpp/README.md b/lang/cpp/README.md index a0b92f21835..fe9d2527e63 100644 --- a/lang/cpp/README.md +++ b/lang/cpp/README.md @@ -35,3 +35,11 @@ cmake -Bbuild -DBUILD_EXAMPLES=ON cmake --build build -j ./build/examples/hello-vortex ``` + +## Check coverage + +This will generate an LCOV directory `coverage`: + +```sh +./gcov-report.sh generate +``` diff --git a/lang/cpp/gcov-report.sh b/lang/cpp/gcov-report.sh new file mode 100755 index 00000000000..eded2782673 --- /dev/null +++ b/lang/cpp/gcov-report.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu +cmake -Bbuild -DBUILD_TESTS=1 -DCMAKE_CXX_FLAGS='-fprofile-arcs -ftest-coverage' +cmake --build build -j +# run tests +ctest --test-dir build --output-on-failure -T Test +# collect coverage +ctest --test-dir build -T Coverage || true + +geninfo build/CMakeFiles/vortex_cxx_shared.dir/ \ + build/tests/CMakeFiles/vortex_cxx_test.dir/ \ + --exclude /usr --exclude build/_deps --exclude tests \ + -b src \ + -o coverage.info +[ $# -gt 0 ] && genhtml coverage.info -o coverage diff --git a/lang/cpp/include/vortex/array.hpp b/lang/cpp/include/vortex/array.hpp index 82c1b5d747d..43b986f4b42 100644 --- a/lang/cpp/include/vortex/array.hpp +++ b/lang/cpp/include/vortex/array.hpp @@ -40,8 +40,8 @@ enum class ValidityType { AllValid = VX_VALIDITY_ALL_VALID, // All items are invalid AllInvalid = VX_VALIDITY_ALL_INVALID, - // Item validity is set in a boolean array: true = valid, false = invalid - Array = VX_VALIDITY_ARRAY, + // Item validity is set from a boolean array: true = valid, false = invalid + FromArray = VX_VALIDITY_ARRAY, }; /** diff --git a/lang/cpp/include/vortex/common.hpp b/lang/cpp/include/vortex/common.hpp index 1248f6d53bd..aad6eb5c3ad 100644 --- a/lang/cpp/include/vortex/common.hpp +++ b/lang/cpp/include/vortex/common.hpp @@ -13,9 +13,35 @@ namespace vortex { struct float16_t { uint16_t bits; - friend bool operator==(float16_t, float16_t) = default; + constexpr friend bool operator==(float16_t, float16_t) = default; // NOLINTNEXTLINE - operator float() const; + constexpr operator float() const { + float result; + const uint32_t sign = (bits >> 15) & 1; + const uint32_t exponent = (bits >> 10) & 0x1F; + const uint32_t mantissa = bits & 0x3FF; + + uint32_t out; + if (exponent == 0x1F) { + out = (sign << 31) | 0x7F800000 | (mantissa << 13); + } else if (exponent == 0) { + if (mantissa == 0) { + out = sign << 31; + } else { + uint32_t m = mantissa; + int e = -1; + do { + m <<= 1; + ++e; + } while ((m & 0x400) == 0); + out = (sign << 31) | ((127 - 15 - e) << 23) | ((m & 0x3FF) << 13); + } + } else { + out = (sign << 31) | ((exponent - 15 + 127) << 23) | (mantissa << 13); + } + memcpy(&result, &out, sizeof(result)); + return result; + } }; static_assert(sizeof(float16_t) == 2 && std::is_trivially_copyable_v); } // namespace vortex diff --git a/lang/cpp/include/vortex/scan.hpp b/lang/cpp/include/vortex/scan.hpp index 7175f494bef..1b8cc2ef7a4 100644 --- a/lang/cpp/include/vortex/scan.hpp +++ b/lang/cpp/include/vortex/scan.hpp @@ -20,8 +20,9 @@ namespace vortex { namespace detail { + // range-for support for Scan and Partition -template +template (Source::*Next)()> class PullRange { public: class iterator { @@ -37,7 +38,7 @@ class PullRange { return *cur_; } iterator &operator++() { - cur_ = Next(src_); + cur_ = (src_->*Next)(); return *this; } void operator++(int) { @@ -55,7 +56,7 @@ class PullRange { explicit PullRange(Source &src) : src_(&src) { } iterator begin() { - return iterator(src_, Next(src_)); + return iterator(src_, (src_->*Next)()); } std::default_sentinel_t end() { return std::default_sentinel; diff --git a/lang/cpp/src/array.cpp b/lang/cpp/src/array.cpp index 11b012449e2..791fbd7d586 100644 --- a/lang/cpp/src/array.cpp +++ b/lang/cpp/src/array.cpp @@ -23,14 +23,14 @@ Validity::Validity(const Validity &other) } Validity::Validity(ValidityType type) : type_(type), array_(nullptr) { - if (type == ValidityType::Array) { + if (type == ValidityType::FromArray) { throw VortexException("Validity(ValidityType) called with ValidityType::Array", ErrorCode::InvalidArgument); } } Validity Validity::from_array(const Array &bools) { - return {ValidityType::Array, vx_array_clone(Access::c_ptr(bools))}; + return {ValidityType::FromArray, vx_array_clone(Access::c_ptr(bools))}; } Validity::Validity(Validity &&other) noexcept : type_(other.type_), array_(other.array_) { @@ -61,7 +61,7 @@ Validity::~Validity() { } Array Validity::array() const { - if (type_ != ValidityType::Array || array_ == nullptr) { + if (type_ != ValidityType::FromArray || array_ == nullptr) { throw VortexException("validity has no backing array", ErrorCode::InvalidArgument); } return Access::adopt(vx_array_clone(array_)); @@ -93,7 +93,7 @@ ValidityBits::ValidityBits(const Session &session, const vx_array *canonical) { case ValidityType::AllInvalid: all_invalid_ = true; return; - case ValidityType::Array: + case ValidityType::FromArray: break; } @@ -163,7 +163,7 @@ Array Array::primitive_raw(vx_ptype ptype, const void *data, size_t len, const V std::optional keep_alive; vx_validity raw {}; raw.type = static_cast(validity.type()); - if (validity.type() == ValidityType::Array) { + if (validity.type() == ValidityType::FromArray) { keep_alive = validity.array(); raw.array = Access::c_ptr(*keep_alive); } @@ -264,7 +264,7 @@ Array make_struct(std::span fields, const Validity &validity) raw.type = static_cast(validity.type()); std::optional keep_alive; - if (validity.type() == ValidityType::Array) { + if (validity.type() == ValidityType::FromArray) { keep_alive = validity.array(); raw.array = Access::c_ptr(*keep_alive); } diff --git a/lang/cpp/src/f16.cpp b/lang/cpp/src/f16.cpp deleted file mode 100644 index de1058cc7d4..00000000000 --- a/lang/cpp/src/f16.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex/common.hpp" - -#include -#include - -namespace vortex { - -#if __STDCPP_FLOAT16_T__ != 1 -float16_t::operator float() const { - float result; - const uint32_t sign = (bits >> 15) & 1; - const uint32_t exponent = (bits >> 10) & 0x1F; - const uint32_t mantissa = bits & 0x3FF; - - uint32_t out; - if (exponent == 0x1F) { - out = (sign << 31) | 0x7F800000 | (mantissa << 13); - } else if (exponent == 0) { - if (mantissa == 0) { - out = sign << 31; - } else { - uint32_t m = mantissa; - int e = -1; - do { - m <<= 1; - ++e; - } while ((m & 0x400) == 0); - out = (sign << 31) | ((127 - 15 - e) << 23) | ((m & 0x3FF) << 13); - } - } else { - out = (sign << 31) | ((exponent - 15 + 127) << 23) | (mantissa << 13); - } - std::memcpy(&result, &out, sizeof(result)); - return result; -} -#endif -} // namespace vortex diff --git a/lang/cpp/tests/CMakeLists.txt b/lang/cpp/tests/CMakeLists.txt new file mode 100644 index 00000000000..16a6e0290b9 --- /dev/null +++ b/lang/cpp/tests/CMakeLists.txt @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +# need this for std::float16_t +set(CMAKE_CXX_STANDARD 23) + +FetchContent_Declare( + magic_enum + GIT_REPOSITORY https://github.com/Neargye/magic_enum.git + GIT_TAG v0.9.7 +) +FetchContent_MakeAvailable(magic_enum) + +file(GLOB TEST_FILES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") + +add_executable(vortex_cxx_test ${TEST_FILES}) +target_include_directories(vortex_cxx_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(vortex_cxx_test PRIVATE + vortex_cxx_shared + Catch2::Catch2WithMain + magic_enum::magic_enum + nanoarrow_shared) + +catch_discover_tests(vortex_cxx_test) diff --git a/lang/cpp/tests/array.cpp b/lang/cpp/tests/array.cpp new file mode 100644 index 00000000000..ef058e8c4bf --- /dev/null +++ b/lang/cpp/tests/array.cpp @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include +#include +#include + +using namespace vortex; +using namespace vortex::expr::ops; + +namespace { +using enum vortex::PType; +using enum ValidityType; + +TEST_CASE("Null array", "[array]") { + Array a = Array::null(1999); + REQUIRE(a.size() == 1999); + REQUIRE(a.nullable()); + REQUIRE(a.has_dtype(DataTypeVariant::Null)); +} + +TEST_CASE("Empty array", "[array]") { + Session session; + + auto empty = Array::primitive({}); + REQUIRE(empty.size() == 0); + REQUIRE(empty.is_primitive(I32)); + + REQUIRE(empty.null_count() == 0); + + auto view = empty.values(session); + REQUIRE(view.size() == 0); + auto values = view.values(); + REQUIRE(values.empty()); +} + +TEST_CASE("Primitive array", "[array]") { + Session session; + + int32_t c_array[3] = {10, 20, 30}; + auto b = Array::primitive(c_array); + + const int32_t const_c_array[3] = {10, 20, 30}; + auto c = Array::primitive(const_c_array); + + const std::array cpp_array = {10, 20, 30}; + auto d = Array::primitive(cpp_array); + + std::vector cpp_vector = {10, 20, 30}; + Array a = Array::primitive(cpp_vector); + + REQUIRE(a.size() == 3); + REQUIRE(a.is_primitive(I32)); + REQUIRE_FALSE(a.nullable()); + REQUIRE(a.null_count() == 0); + + auto view = a.values(session); + REQUIRE(view.size() == 3); + REQUIRE(std::equal(view.values().begin(), view.values().end(), cpp_vector.begin())); + REQUIRE_FALSE(view.is_null(1)); +} + +TEST_CASE("values with wrong type", "[array]") { + Session session; + std::vector data = {1}; + Array a = Array::primitive(data); + REQUIRE_THROWS_AS(a.values(session), VortexException); +} + +TEST_CASE("Validity from a boolean mask", "[array]") { + Session session; + std::vector data = {10, 20, 30}; + std::vector mask_bytes = {1, 0, 1}; + + Array mask_u8 = Array::primitive(std::span(mask_bytes)); + Array mask = mask_u8.apply(expr::root() == expr::lit(1)); + + Array a = Array::primitive(std::span(data), Validity::from_array(mask)); + REQUIRE(a.nullable()); + REQUIRE(a.null_count() == 1); + + auto view = a.values(session); + REQUIRE_FALSE(view.is_null(0)); + REQUIRE(view.is_null(1)); + REQUIRE_FALSE(view.is_null(2)); + REQUIRE(view.values()[0] == 10); + REQUIRE(view.values()[2] == 30); + + Validity validity = a.validity(); + REQUIRE(validity.type() == FromArray); + REQUIRE(validity.array().size() == 3); +} + +TEST_CASE("AllInvalid", "[array]") { + Session session; + std::vector data = {1, 2}; + Array a = Array::primitive(std::span(data), AllInvalid); + REQUIRE(a.null_count() == 2); + auto view = a.values(session); + REQUIRE(view.is_null(0)); + REQUIRE(view.is_null(1)); +} + +TEST_CASE("make_struct and fields", "[array]") { + Array empty = make_struct({}); + REQUIRE(empty.size() == 0); + REQUIRE(empty.has_dtype(DataTypeVariant::Struct)); + REQUIRE(empty.dtype().fields().size() == 0); + + std::vector ages = {10, 20, 30}; + std::vector heights = {150, 160, 170}; + + Array s = make_struct({ + {"age", Array::primitive(ages)}, + {"height", Array::primitive(heights, AllValid)}, + }); + + REQUIRE(s.size() == 3); + REQUIRE(s.has_dtype(DataTypeVariant::Struct)); + REQUIRE(s.dtype().fields().size() == 2); + + Array by_index = s.field(0); + REQUIRE(by_index.is_primitive(U8)); + + Session session; + Array by_name = s.field("height"); + REQUIRE(by_name.is_primitive(U16)); + auto view = by_name.values(session); + REQUIRE(view.values()[2] == 170); + + REQUIRE_THROWS_AS(s.field(2), VortexException); + REQUIRE_THROWS_AS(s.field("nope"), VortexException); + + std::vector fields_vec; + fields_vec.emplace_back("age", Array::primitive(ages)); + Array other = make_struct(fields_vec); + REQUIRE(other.size() == 3); + REQUIRE(other.has_dtype(DataTypeVariant::Struct)); +} + +TEST_CASE("Mismatched field length", "[array]") { + std::vector a = {1, 2}; + std::vector b = {1, 2, 3}; + REQUIRE_THROWS_AS(make_struct({ + {"a", Array::primitive(a)}, + {"b", Array::primitive(b)}, + }), + VortexException); +} + +TEST_CASE("Slice", "[array]") { + Session session; + std::vector data = {0, 1, 2, 3, 4, 5}; + Array a = Array::primitive(data); + Array sliced = a.slice(2, 5); + REQUIRE(sliced.size() == 3); + auto view = sliced.values(session); + REQUIRE(view.values()[0] == 2); + REQUIRE(view.values()[2] == 4); + + REQUIRE_THROWS_AS(a.slice(2, 100), VortexException); +} + +TEST_CASE("Error with a code", "[array]") { + std::vector data = {0}; + Array a = Array::primitive(data); + try { + (void)a.slice(2, 100); + FAIL("expected exception"); + } catch (const VortexException &e) { + REQUIRE_FALSE(std::string(e.what()).empty()); + (void)e.code(); + } +} +} // namespace diff --git a/lang/cpp/tests/arrow.cpp b/lang/cpp/tests/arrow.cpp new file mode 100644 index 00000000000..8675d816067 --- /dev/null +++ b/lang/cpp/tests/arrow.cpp @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include + +#include +#include + +typedef struct ArrowSchema FFI_ArrowSchema; +typedef struct ArrowArray FFI_ArrowArray; +typedef struct ArrowArrayStream FFI_ArrowArrayStream; +#define USE_OWN_ARROW 1 + +#include + +#include "common.hpp" + +using namespace vortex; +using vortex_test::sample_dtype; +using vortex_test::TempPath; +using vortex_test::write_sample; + +namespace { +using enum vortex::PType; + +TEST_CASE("dtype to ArrowSchema", "[arrow]") { + DataType d = sample_dtype(); + ArrowSchema schema = d.to_arrow(); + + nanoarrow::UniqueSchema unique_schema; + ArrowSchemaMove(&schema, unique_schema.get()); + REQUIRE(unique_schema->format != nullptr); + REQUIRE(unique_schema->n_children == 2); +} + +TEST_CASE("dtype from ArrowSchema", "[arrow]") { + nanoarrow::UniqueSchema schema; + REQUIRE(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_STRUCT) == NANOARROW_OK); + REQUIRE(ArrowSchemaAllocateChildren(schema.get(), 1) == NANOARROW_OK); + REQUIRE(ArrowSchemaInitFromType(schema->children[0], NANOARROW_TYPE_INT64) == NANOARROW_OK); + REQUIRE(ArrowSchemaSetName(schema->children[0], "n") == NANOARROW_OK); + + ArrowSchema raw = {}; + ArrowSchemaMove(schema.get(), &raw); + DataType d = DataType::from_arrow(&raw); + REQUIRE(d.variant() == DataTypeVariant::Struct); + const std::vector fields = d.fields(); + REQUIRE(fields.size() == 1); + REQUIRE(fields[0].name == "n"); + REQUIRE(fields[0].dtype.primitive_type() == I64); +} + +TEST_CASE("Import Arrow array as Vortex array", "[arrow]") { + Session session; + nanoarrow::UniqueSchema schema; + REQUIRE(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_STRUCT) == NANOARROW_OK); + REQUIRE(ArrowSchemaAllocateChildren(schema.get(), 1) == NANOARROW_OK); + REQUIRE(ArrowSchemaInitFromType(schema->children[0], NANOARROW_TYPE_INT32) == NANOARROW_OK); + REQUIRE(ArrowSchemaSetName(schema->children[0], "a") == NANOARROW_OK); + + nanoarrow::UniqueArray arr; + REQUIRE(ArrowArrayInitFromSchema(arr.get(), schema.get(), nullptr) == NANOARROW_OK); + REQUIRE(ArrowArrayStartAppending(arr.get()) == NANOARROW_OK); + for (int i : {10, 20, 30}) { + REQUIRE(ArrowArrayAppendInt(arr->children[0], i) == NANOARROW_OK); + REQUIRE(ArrowArrayFinishElement(arr.get()) == NANOARROW_OK); + } + REQUIRE(ArrowArrayFinishBuildingDefault(arr.get(), nullptr) == NANOARROW_OK); + + ArrowArray raw_arr = {}; + ArrowSchema raw_schema = {}; + ArrowArrayMove(arr.get(), &raw_arr); + ArrowSchemaMove(schema.get(), &raw_schema); + + Array vx = Array::from_arrow(&raw_arr, &raw_schema, false); + REQUIRE(vx.size() == 3); + REQUIRE(vx.has_dtype(DataTypeVariant::Struct)); + + Array a = vx.field(0); + REQUIRE(a.is_primitive(I32)); + auto view = a.values(session); + REQUIRE(view.values()[0] == 10); + REQUIRE(view.values()[2] == 30); +} + +TEST_CASE("Scan partition to ArrowArrayStream", "[arrow]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan(); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + + ArrowStream vx_stream = std::move(partition.value()).into_arrow_stream(); + nanoarrow::UniqueArrayStream owned; + ArrowArrayStreamMove(vx_stream.raw(), owned.get()); + + nanoarrow::UniqueSchema schema; + ArrowError err {}; + REQUIRE(ArrowArrayStreamGetSchema(owned.get(), schema.get(), &err) == NANOARROW_OK); + REQUIRE(schema->n_children == 2); + + size_t rows = 0; + while (true) { + nanoarrow::UniqueArray chunk; + int rc = owned->get_next(owned.get(), chunk.get()); + REQUIRE(rc == NANOARROW_OK); + if (chunk->release == nullptr) { + break; + } + rows += chunk->length; + } + REQUIRE(rows == vortex_test::SAMPLE_ROWS); +} +} // namespace diff --git a/lang/cpp/tests/common.hpp b/lang/cpp/tests/common.hpp new file mode 100644 index 00000000000..b1b3d982ffa --- /dev/null +++ b/lang/cpp/tests/common.hpp @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include "vortex/array.hpp" +#include +#include +#include +#include + +#include +#include + +namespace vortex_test { + +namespace fs = std::filesystem; +using namespace vortex; + +class TempPath { +public: + TempPath() = default; + explicit TempPath(fs::path p) : path_(std::move(p)) { + } + + TempPath(const TempPath &) = delete; + TempPath &operator=(const TempPath &) = delete; + + TempPath(TempPath &&other) noexcept : path_(std::move(other.path_)) { + other.path_.clear(); + } + TempPath &operator=(TempPath &&other) noexcept { + if (this != &other) { + reset(); + path_ = std::move(other.path_); + other.path_.clear(); + } + return *this; + } + + ~TempPath() { + reset(); + } + + const fs::path &path() const noexcept { + return path_; + } + std::string string() const { + return path_.string(); + } + + static TempPath unique() { + auto dir = fs::temp_directory_path() / "vortex_cxx_test"; + fs::create_directories(dir); + std::string name = std::to_string(std::random_device {}()) + ".vortex"; + return TempPath {dir / name}; + } + +private: + void reset() noexcept { + if (!path_.empty()) { + std::error_code ec; + fs::remove(path_, ec); + } + } + + fs::path path_; +}; + +inline DataType sample_dtype() { + return dtype::struct_({ + {"age", dtype::uint8()}, + {"height", dtype::uint16(dtype::Nullable)}, + }); +} + +constexpr size_t SAMPLE_ROWS = 100; + +inline std::vector sample_ages() { + std::vector buf(SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + buf[i] = static_cast(i); + } + return buf; +} + +inline std::vector sample_heights() { + std::vector buf(SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + buf[i] = static_cast((i + 1) % 200); + } + return buf; +} + +inline Array sample_array() { + auto ages = sample_ages(); + auto heights = sample_heights(); + return make_struct({ + {"age", Array::primitive(ages)}, + {"height", Array::primitive(heights, ValidityType::AllValid)}, + }); +} + +inline TempPath write_sample(const Session &session) { + TempPath path = TempPath::unique(); + Writer writer = Writer::open(session, path.string(), sample_dtype()); + writer.push(sample_array()); + writer.finish(); + return path; +} +} // namespace vortex_test diff --git a/lang/cpp/tests/dtype.cpp b/lang/cpp/tests/dtype.cpp new file mode 100644 index 00000000000..1b5e3573092 --- /dev/null +++ b/lang/cpp/tests/dtype.cpp @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include "vortex/dtype.hpp" +#include +#include + +using namespace vortex; + +namespace { +using enum vortex::PType; + +TEST_CASE("Null dtype", "[dtype]") { + auto d = dtype::null(); + REQUIRE(d.variant() == DataTypeVariant::Null); + REQUIRE(d.nullable()); +} + +TEST_CASE("Decimal dtype", "[dtype]") { + auto d = dtype::decimal(5, 2, false); + REQUIRE(d.variant() == DataTypeVariant::Decimal); + REQUIRE(d.decimal_precision() == 5); + REQUIRE(d.decimal_scale() == 2); + REQUIRE_FALSE(d.nullable()); + + REQUIRE_THROWS_AS(d.fields(), VortexException); + REQUIRE_THROWS_AS(d.list_element(), VortexException); +} + +TEST_CASE("copy dtype", "[dtype]") { + auto d = dtype::int32(true); + DataType d2 = d; + REQUIRE(d2.variant() == DataTypeVariant::Primitive); + REQUIRE(d2.primitive_type() == I32); + REQUIRE(d2.nullable()); + REQUIRE(d.variant() == DataTypeVariant::Primitive); +} + +TEST_CASE("list dtype", "[dtype]") { + auto d = dtype::list(dtype::float64(), dtype::Nullable); + REQUIRE(d.variant() == DataTypeVariant::List); + REQUIRE(d.nullable()); + REQUIRE(d.list_element().primitive_type() == F64); + + auto fsl = dtype::fixed_size_list(dtype::int16(), 4); + REQUIRE(fsl.variant() == DataTypeVariant::FixedSizeList); + REQUIRE(fsl.fixed_size_list_size() == 4); + REQUIRE(fsl.fixed_size_list_element().primitive_type() == I16); +} + +TEST_CASE("Struct DataType", "[dtype]") { + DataType d = dtype::struct_({ + {"col1", dtype::uint8()}, + {"col2", dtype::binary(dtype::Nullable)}, + }); + + REQUIRE(d.variant() == DataTypeVariant::Struct); + REQUIRE_FALSE(d.nullable()); + const std::vector fields = d.fields(); + REQUIRE(fields.size() == 2); + REQUIRE(fields[0].name == "col1"); + REQUIRE(fields[1].name == "col2"); + REQUIRE(fields[0].dtype.primitive_type() == U8); + REQUIRE(fields[1].dtype.variant() == DataTypeVariant::Binary); + REQUIRE(fields[1].dtype.nullable()); + + std::vector fields_vec; + fields_vec.emplace_back("col1", dtype::uint8()); + fields_vec.emplace_back("col2", dtype::utf8()); + d = dtype::struct_(fields_vec, dtype::Nullable); + + REQUIRE(d.variant() == DataTypeVariant::Struct); + REQUIRE(d.nullable()); + const std::vector built = d.fields(); + REQUIRE(built.size() == 2); + REQUIRE(built[1].name == "col2"); + + fields_vec = {}; + fields_vec.emplace_back("\xFF\xFE", dtype::uint8()); + REQUIRE_THROWS_AS(dtype::struct_(fields_vec), VortexException); +} +} // namespace diff --git a/lang/cpp/tests/enum_sizes.cpp b/lang/cpp/tests/enum_sizes.cpp new file mode 100644 index 00000000000..a100fa3b583 --- /dev/null +++ b/lang/cpp/tests/enum_sizes.cpp @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include + +#include +#include + +using magic_enum::enum_count; + +static_assert(enum_count() == enum_count()); +static_assert(enum_count() == enum_count()); +static_assert(enum_count() == enum_count()); +static_assert(enum_count() == enum_count()); +static_assert(enum_count() == enum_count()); +static_assert(enum_count() == enum_count()); diff --git a/lang/cpp/tests/expression.cpp b/lang/cpp/tests/expression.cpp new file mode 100644 index 00000000000..1a55364df02 --- /dev/null +++ b/lang/cpp/tests/expression.cpp @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include + +#include +#include +#include + +using namespace vortex; + +namespace { +using enum vortex::PType; + +TEST_CASE("Expression copy", "[expr]") { + Expression clone = expr::root(); + { + Expression orig = expr::col("age"); + clone = orig; + } + Expression another = clone; + (void)another; + SUCCEED(); +} + +TEST_CASE("Apply root()", "[expr]") { + Session session; + std::vector data = {10, 20, 30, 40, 50}; + Array array = Array::primitive(data); + + Array applied = array.apply(expr::root()); + + REQUIRE(applied.size() == data.size()); + REQUIRE(applied.is_primitive(I32)); + auto values = applied.values(session); + for (size_t i = 0; i < data.size(); ++i) { + REQUIRE(values.values()[i] == data[i]); + } +} + +TEST_CASE("Apply projection", "[expr]") { + Session session; + std::vector ages = {10, 20, 30}; + std::vector heights = {150, 160, 170}; + + Array struct_arr = make_struct({ + {"age", Array::primitive(ages)}, + {"height", Array::primitive(heights)}, + }); + + Array projected = struct_arr.apply(expr::col("age")); + REQUIRE(projected.size() == ages.size()); + REQUIRE(projected.is_primitive(U8)); + auto values = projected.values(session); + for (size_t i = 0; i < ages.size(); ++i) { + REQUIRE(values.values()[i] == ages[i]); + } +} + +TEST_CASE("Apply arithmetic", "[expr]") { + Session session; + std::vector data = {1, 2, 3, 4, 5}; + Array array = Array::primitive(data); + + Expression e = expr::add(expr::root(), expr::lit(10)); + Array applied = array.apply(e); + + REQUIRE(applied.is_primitive(I32)); + auto values = applied.values(session); + for (size_t i = 0; i < data.size(); ++i) { + REQUIRE(values.values()[i] == data[i] + 10); + } +} + +TEST_CASE("Operator overloading", "[expr]") { + using namespace vortex::expr::ops; + Session session; + std::vector data = {1, 2, 2, 7}; + Array array = Array::primitive(data); + + Array applied = array.apply(expr::root() == expr::lit(2)); + auto bits = applied.bools(session); + REQUIRE(bits.size() == data.size()); + REQUIRE_FALSE(bits.value(0)); + REQUIRE(bits.value(1)); + REQUIRE(bits.value(2)); + REQUIRE_FALSE(bits.value(3)); +} + +TEST_CASE("Apply error", "[expr]") { + std::vector data = {1, 2, 3}; + Array array = Array::primitive(data); + + Expression bad = expr::add(expr::root(), expr::lit(1)); + REQUIRE_THROWS_AS(array.apply(bad), VortexException); +} + +TEST_CASE("Empty conjunction", "[expr]") { + REQUIRE_THROWS_AS(expr::and_all({}), VortexException); + REQUIRE_THROWS_AS(expr::or_all({}), VortexException); +} +} // namespace diff --git a/lang/cpp/tests/float16_t.cpp b/lang/cpp/tests/float16_t.cpp new file mode 100644 index 00000000000..d0197e5d551 --- /dev/null +++ b/lang/cpp/tests/float16_t.cpp @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex/dtype.hpp" +#include +#include + +using namespace vortex; + +namespace { +TEST_CASE("float16_t to_float", "[float]") { + REQUIRE(float(1.0f16) == 1.0F); + REQUIRE(float(-2.0f16) == -2.0F); + REQUIRE(float(0.0f) == 0.0F); + REQUIRE(float(-0.0f) == -0.0F); +} + +TEST_CASE("F16 scalar", "[scalar]") { + float16_t float16t = 1.0f16; + Scalar scalar = scalar::of(float16t); + REQUIRE(scalar.dtype().variant() == DataTypeVariant::Primitive); + REQUIRE(scalar.dtype().primitive_type() == vortex::PType::F16); +} +} // namespace diff --git a/lang/cpp/tests/float16_t_compat.cpp b/lang/cpp/tests/float16_t_compat.cpp new file mode 100644 index 00000000000..506fd84ca63 --- /dev/null +++ b/lang/cpp/tests/float16_t_compat.cpp @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include +#include +// This is UB but no other way to test compatibility macro, fine +// as we're running tests +#undef __STDCPP_FLOAT16_T__ +#include +#include + +using namespace vortex; + +namespace { +TEST_CASE("float16_t to_float (compatibility)", "[float]") { + REQUIRE(float(float16_t {0x3C00}) == 1.0F); + REQUIRE(float(float16_t {0xC000}) == -2.0F); + REQUIRE(float(float16_t {0}) == 0.0F); + REQUIRE(float(float16_t {0x8000}) == -0.0F); + REQUIRE(float(float16_t {0x7C00}) == std::numeric_limits::infinity()); + REQUIRE(float(float16_t {0xFC00}) == -std::numeric_limits::infinity()); + REQUIRE(std::fpclassify(float(float16_t {0x7E01})) == FP_NAN); + // Denormalized float16_t always gets to normal floats on conversion to float + REQUIRE(std::fpclassify(float(float16_t {0x0001})) == FP_NORMAL); + REQUIRE(std::fpclassify(float(float16_t {0x83FF})) == FP_NORMAL); +} + +TEST_CASE("F16 scalar (compatibility)", "[scalar]") { + const float16_t float16t {0x3C00}; + Scalar scalar = scalar::of(float16t); + REQUIRE(scalar.dtype().variant() == DataTypeVariant::Primitive); + REQUIRE(scalar.dtype().primitive_type() == vortex::PType::F16); +} +} // namespace diff --git a/lang/cpp/tests/scalar.cpp b/lang/cpp/tests/scalar.cpp new file mode 100644 index 00000000000..27ff73f656a --- /dev/null +++ b/lang/cpp/tests/scalar.cpp @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include "vortex/error.hpp" +#include +#include +#include + +using namespace vortex; +using namespace std::string_view_literals; + +namespace { +using enum vortex::PType; + +TEST_CASE("Boolean scalar", "[scalar]") { + Scalar s = scalar::of(true); + REQUIRE_FALSE(s.is_null()); + REQUIRE(s.dtype().variant() == DataTypeVariant::Bool); +} + +TEST_CASE("Integer scalars", "[scalar]") { + REQUIRE(scalar::of(42).dtype().primitive_type() == U8); + REQUIRE(scalar::of(42).dtype().primitive_type() == U16); + REQUIRE(scalar::of(42).dtype().primitive_type() == U32); + REQUIRE(scalar::of(42).dtype().primitive_type() == U64); + REQUIRE(scalar::of(-1).dtype().primitive_type() == I8); + REQUIRE(scalar::of(-1).dtype().primitive_type() == I16); + REQUIRE(scalar::of(-1).dtype().primitive_type() == I32); + REQUIRE(scalar::of(-1).dtype().primitive_type() == I64); +} + +TEST_CASE("Float scalars", "[scalar]") { + REQUIRE(scalar::of(1.5F).dtype().primitive_type() == F32); + REQUIRE(scalar::of(1.5).dtype().primitive_type() == F64); + REQUIRE(scalar::of(float16_t {0x3C00}).dtype().primitive_type() == F16); +} + +TEST_CASE("Nullable scalars", "[scalar]") { + Scalar s = scalar::of(0, true); + REQUIRE(s.dtype().nullable()); + REQUIRE_FALSE(s.is_null()); +} + +TEST_CASE("Null scalar", "[scalar]") { + Scalar s = scalar::null(dtype::int32(true)); + REQUIRE(s.is_null()); + REQUIRE(s.dtype().variant() == DataTypeVariant::Primitive); +} + +TEST_CASE("UTF-8 scalar", "[scalar]") { + Scalar s = scalar::of("hello"sv); + REQUIRE_FALSE(s.is_null()); + REQUIRE(s.dtype().variant() == DataTypeVariant::Utf8); + + REQUIRE_FALSE(scalar::of(""sv).is_null()); + s = scalar::of("Широкая строка"sv); + REQUIRE(s.dtype().variant() == DataTypeVariant::Utf8); + + REQUIRE_THROWS_AS(scalar::of("\xFF\xFE"sv), VortexException); +} + +TEST_CASE("Binary scalar", "[scalar]") { + const std::byte bytes[] = {std::byte {1}, std::byte {2}, std::byte {0}, std::byte {4}}; + Scalar s = scalar::of(std::span {bytes}); + REQUIRE_FALSE(s.is_null()); + REQUIRE(s.dtype().variant() == DataTypeVariant::Binary); +} + +TEST_CASE("Decimal scalars", "[scalar]") { + Scalar d8 = scalar::decimal(56, 5, 2); + REQUIRE(d8.dtype().variant() == DataTypeVariant::Decimal); + REQUIRE(d8.dtype().decimal_precision() == 5); + REQUIRE(d8.dtype().decimal_scale() == 2); + + Scalar d16 = scalar::decimal(int16_t(1234), 5, 2); + REQUIRE(d16.dtype().variant() == DataTypeVariant::Decimal); + REQUIRE(d16.dtype().decimal_precision() == 5); + REQUIRE(d16.dtype().decimal_scale() == 2); + + Scalar d32 = scalar::decimal(int32_t(1234), 5, 2); + REQUIRE(d32.dtype().variant() == DataTypeVariant::Decimal); + REQUIRE(d32.dtype().decimal_precision() == 5); + REQUIRE(d32.dtype().decimal_scale() == 2); + + Scalar d64 = scalar::decimal(99999, 12, 3); + REQUIRE(d64.dtype().variant() == DataTypeVariant::Decimal); + REQUIRE(d64.dtype().decimal_precision() == 12); + REQUIRE(d64.dtype().decimal_scale() == 3); +} + +TEST_CASE("Copy scalar", "[scalar]") { + Scalar a = scalar::of(42); + Scalar b = a; + REQUIRE(a.dtype().primitive_type() == I64); + REQUIRE(b.dtype().primitive_type() == I64); + + Scalar c = scalar::of(1); + c = b; + REQUIRE(c.dtype().primitive_type() == I64); +} +} // namespace diff --git a/lang/cpp/tests/scan.cpp b/lang/cpp/tests/scan.cpp new file mode 100644 index 00000000000..82854e11f2a --- /dev/null +++ b/lang/cpp/tests/scan.cpp @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include +#include +#include +#include +#include + +#include "common.hpp" +#include "vortex/estimate.hpp" + +using namespace vortex; +using Catch::Matchers::ContainsSubstring; +using vortex_test::SAMPLE_ROWS; +using vortex_test::TempPath; +using vortex_test::write_sample; + +namespace { +using enum vortex::PType; + +TEST_CASE("Non-existent data source", "[datasource]") { + Session session; + + try { + DataSource::open(session, {"nonexistent"}); + FAIL("expected exception"); + } catch (const VortexException &e) { + REQUIRE_THAT(std::string(e.what()), ContainsSubstring("No files matched")); + } +} + +TEST_CASE("Read dtype and row count", "[datasource]") { + Session session; + TempPath path = write_sample(session); + + std::array paths = {path.string()}; + + DataSource other = DataSource::open(session, paths); + DataSource ds(other); + + Estimate row_count = ds.row_count(); + REQUIRE(row_count.type() == EstimateType::Exact); + REQUIRE(row_count.value() == SAMPLE_ROWS); + + DataType dt = ds.dtype(); + REQUIRE(dt.variant() == DataTypeVariant::Struct); + const std::vector fields = dt.fields(); + REQUIRE(fields.size() == 2); + REQUIRE(fields[0].name == "age"); + REQUIRE(fields[1].name == "height"); +} + +void verify_age_field(const Session &session, + const Array &age, + uint8_t first = 0, + size_t rows = SAMPLE_ROWS) { + REQUIRE(age.is_primitive(U8)); + REQUIRE(age.size() == rows); + auto view = age.values(session); + for (size_t i = 0; i < rows; ++i) { + REQUIRE(view.values()[i] == static_cast(first + i)); + } +} + +void verify_sample_array(const Session &session, const Array &array) { + REQUIRE(array.size() == SAMPLE_ROWS); + REQUIRE(array.has_dtype(DataTypeVariant::Struct)); + verify_age_field(session, array.field(0)); + + Array height = array.field(1); + REQUIRE(height.is_primitive(U16)); + auto view = height.values(session); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + REQUIRE(view.values()[i] == (i + 1) % 200); + } + + REQUIRE_THROWS_AS(array.field(2), VortexException); +} + +TEST_CASE("Basic scan", "[scan]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan(); + REQUIRE(scan.partition_count().type() == EstimateType::Exact); + REQUIRE(scan.partition_count().value() == 1); + REQUIRE(scan.dtype().variant() == DataTypeVariant::Struct); + + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + + Estimate rows = partition->row_count(); + REQUIRE(rows.type() == EstimateType::Exact); + REQUIRE(rows.value() == SAMPLE_ROWS); + + REQUIRE_FALSE(scan.next_partition().has_value()); + + auto array = partition->next(); + REQUIRE(array.has_value()); + REQUIRE_FALSE(partition->next().has_value()); + + verify_sample_array(session, *array); +} + +TEST_CASE("Range-for over partitions and batches", "[scan]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan(); + size_t partitions = 0; + size_t rows = 0; + for (Partition &partition : scan.partitions()) { + ++partitions; + for (Array &batch : partition.batches()) { + rows += batch.size(); + } + } + REQUIRE(partitions == 1); + REQUIRE(rows == SAMPLE_ROWS); +} + +TEST_CASE("Scan from buffer", "[scan]") { + Session session; + TempPath path = write_sample(session); + + std::ifstream file(path.path(), std::ios::binary | std::ios::ate); + const auto size = file.tellg(); + file.seekg(0, std::ios::beg); + std::vector buffer(static_cast(size)); + REQUIRE(file.read(reinterpret_cast(buffer.data()), size)); + + DataSource ds = DataSource::from_buffer(session, buffer); + Scan scan = ds.scan(); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + verify_sample_array(session, *array); +} + +TEST_CASE("Multiple paths", "[scan]") { + Session session; + TempPath first = write_sample(session); + TempPath second = write_sample(session); + + const std::vector paths = {first.string(), second.string()}; + DataSource ds = DataSource::open(session, paths); + REQUIRE(ds.row_count().value_or(0) >= SAMPLE_ROWS); +} + +TEST_CASE("Multithreaded scan", "[scan]") { + Session session; + + constexpr size_t NUM_FILES = 8; + std::vector paths; + std::vector path_strings; + for (size_t i = 0; i < NUM_FILES; ++i) { + paths.push_back(write_sample(session)); + path_strings.push_back(paths.back().string()); + } + + DataSource ds = DataSource::open(session, path_strings); + Scan scan = ds.scan(); + + std::vector threads; + threads.reserve(NUM_FILES); + std::vector rows(NUM_FILES, 0); + + for (size_t i = 0; i < NUM_FILES; ++i) { + threads.emplace_back([&, i] { + while (auto partition = scan.next_partition()) { + while (auto batch = partition->next()) { + rows[i] += batch->size(); + } + } + }); + } + for (auto &t : threads) { + t.join(); + } + + size_t total = 0; + for (size_t n : rows) { + total += n; + } + REQUIRE(total == NUM_FILES * SAMPLE_ROWS); +} + +TEST_CASE("Project field", "[projection]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan({.projection = expr::col("age")}); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + verify_age_field(session, *array); +} + +TEST_CASE("Project none fields", "[projection]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan({.projection = expr::root().select({})}); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + REQUIRE(array->dtype().fields().empty()); +} + +TEST_CASE("Project multiple fields", "[projection]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan({.projection = expr::root().select({"age", "height"})}); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + verify_age_field(session, array->field("age")); +} + +TEST_CASE("Filter age", "[filter]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + constexpr uint8_t threshold = 50; + Scan scan = ds.scan({ + .filter = expr::gte(expr::col("age"), expr::lit(threshold)), + }); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + + REQUIRE(array->size() == SAMPLE_ROWS - threshold); + verify_age_field(session, array->field(0), threshold, SAMPLE_ROWS - threshold); +} + +TEST_CASE("Filter invalid values", "[filter]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan({.filter = expr::col("age").is_null()}); + auto partition = scan.next_partition(); + REQUIRE(!partition.has_value()); +} + +TEST_CASE("Filter with operators", "[filter]") { + using namespace vortex::expr::ops; + Session session; + TempPath path = write_sample(session); + const std::string path_string = path.string(); + std::array paths = {path_string}; + DataSource ds = DataSource::open(session, paths); + + Scan scan1 = ds.scan({ + .filter = expr::col("age") >= expr::lit(90) && expr::col("age") < expr::lit(95), + }); + + Scan scan2 = ds.scan({ + .filter = !(expr::col("age") < expr::lit(90)) && expr::col("age") < expr::lit(95), + }); + + Scan scans[2] = {std::move(scan1), std::move(scan2)}; + for (Scan &scan : scans) { + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + REQUIRE(array->size() == 5); + } +} + +TEST_CASE("Type-mismatched filter", "[filter]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan({ + .filter = expr::eq(expr::col("age"), expr::lit(67)), + }); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + REQUIRE_THROWS_AS(partition->next(), VortexException); +} + +TEST_CASE("Row range and limit", "[scan]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan({ + .row_range = RowRange {10, 25}, + .limit = 5, + }); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + + REQUIRE(array->size() == 5); + verify_age_field(session, array->field(0), 10, 5); +} + +TEST_CASE("Selection", "[scan]") { + using enum Selection::Kind; + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource::open(session, {path.string()}); + + Scan scan = ds.scan({ + .selection = {{Include, {3, 5, 8}}}, + }); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition->next(); + REQUIRE(array.has_value()); + REQUIRE(array->size() == 3); + + auto ages = array->field(0).values(session); + REQUIRE(ages.values()[0] == 3); + REQUIRE(ages.values()[1] == 5); + REQUIRE(ages.values()[2] == 8); + + scan = ds.scan({ + .selection = {{Exclude, {0, 1, 2}}}, + }); + partition = scan.next_partition(); + REQUIRE(partition.has_value()); + array = partition->next(); + REQUIRE(array.has_value()); + REQUIRE(array->size() == 97); + + ages = array->field(0).values(session); + REQUIRE(ages.values()[0] == 3); +} +} // namespace diff --git a/lang/cpp/tests/session.cpp b/lang/cpp/tests/session.cpp new file mode 100644 index 00000000000..e8a3212a721 --- /dev/null +++ b/lang/cpp/tests/session.cpp @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include + +using namespace vortex; + +namespace { +TEST_CASE("Session copy and move", "[session]") { + const Session s; + Session other; + other = s; + Session moved; + moved = std::move(s); +} +} // namespace diff --git a/lang/cpp/tests/string_binary.cpp b/lang/cpp/tests/string_binary.cpp new file mode 100644 index 00000000000..16ddb7eaa3c --- /dev/null +++ b/lang/cpp/tests/string_binary.cpp @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include + +#include +#include + +typedef struct ArrowSchema FFI_ArrowSchema; +typedef struct ArrowArray FFI_ArrowArray; +typedef struct ArrowArrayStream FFI_ArrowArrayStream; +#define USE_OWN_ARROW 1 + +#include +#include +#include +#include + +#include "common.hpp" + +using namespace vortex; +using namespace std::string_view_literals; +using vortex_test::TempPath; + +namespace { + +Array strings_from_arrow(std::span values, bool with_null) { + nanoarrow::UniqueSchema schema; + REQUIRE(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_STRING) == NANOARROW_OK); + nanoarrow::UniqueArray arr; + REQUIRE(ArrowArrayInitFromSchema(arr.get(), schema.get(), nullptr) == NANOARROW_OK); + REQUIRE(ArrowArrayStartAppending(arr.get()) == NANOARROW_OK); + for (const auto &value : values) { + const ArrowStringView view {value.data(), static_cast(value.size())}; + REQUIRE(ArrowArrayAppendString(arr.get(), view) == NANOARROW_OK); + } + if (with_null) { + REQUIRE(ArrowArrayAppendNull(arr.get(), 1) == NANOARROW_OK); + } + REQUIRE(ArrowArrayFinishBuildingDefault(arr.get(), nullptr) == NANOARROW_OK); + + ArrowArray raw_arr = {}; + ArrowSchema raw_schema = {}; + ArrowArrayMove(arr.get(), &raw_arr); + ArrowSchemaMove(schema.get(), &raw_schema); + return Array::from_arrow(&raw_arr, &raw_schema, true); +} + +Array bytes_from_arrow(std::span values) { + nanoarrow::UniqueSchema schema; + REQUIRE(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_BINARY) == NANOARROW_OK); + nanoarrow::UniqueArray arr; + REQUIRE(ArrowArrayInitFromSchema(arr.get(), schema.get(), nullptr) == NANOARROW_OK); + REQUIRE(ArrowArrayStartAppending(arr.get()) == NANOARROW_OK); + for (const auto &value : values) { + const ArrowBufferView view {{reinterpret_cast(value.data())}, + static_cast(value.size())}; + REQUIRE(ArrowArrayAppendBytes(arr.get(), view) == NANOARROW_OK); + } + REQUIRE(ArrowArrayFinishBuildingDefault(arr.get(), nullptr) == NANOARROW_OK); + + ArrowArray raw_arr = {}; + ArrowSchema raw_schema = {}; + ArrowArrayMove(arr.get(), &raw_arr); + ArrowSchemaMove(schema.get(), &raw_schema); + return Array::from_arrow(&raw_arr, &raw_schema, true); +} + +TEST_CASE("String view over utf8 array", "[strings]") { + Session session; + + const std::string long1(40, 'x'); + const std::vector values = {"short"sv, "Широкая строка"sv, long1, ""sv}; + Array array = strings_from_arrow(values, true); + + StringView view = array.strings(session); + REQUIRE(view.size() == values.size() + 1); + for (size_t i = 0; i < values.size(); ++i) { + REQUIRE_FALSE(view.is_null(i)); + REQUIRE(view[i] == values[i]); + } + REQUIRE(view.is_null(values.size())); + REQUIRE_THROWS_AS(view[view.size()], VortexException); +} + +TEST_CASE("Bytes view over binary array", "[strings]") { + Session session; + + // Includes a NUL byte + const std::vector values = {std::string_view {"abc\0def", 7}, "ffff"sv}; + Array array = bytes_from_arrow(values); + + BytesView view = array.bytes(session); + REQUIRE(view.size() == 2); + for (size_t i = 0; i < values.size(); ++i) { + const auto bytes = view[i]; + REQUIRE(std::string_view(reinterpret_cast(bytes.data()), bytes.size()) == values[i]); + } +} + +TEST_CASE("Strings roundtrip", "[strings]") { + Session session; + TempPath path = TempPath::unique(); + + const std::string long1(64, 'y'); + const std::vector values = {"inlined"sv, long1}; + Array strings = strings_from_arrow(values, false); + + Writer writer = + Writer::open(session, path.string(), dtype::struct_({{"s", dtype::utf8(dtype::Nullable)}})); + writer.push(make_struct({{"s", strings}})); + writer.finish(); + + DataSource ds = DataSource::open(session, {path.string()}); + Scan scan = ds.scan(); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto batch = partition->next(); + REQUIRE(batch.has_value()); + + StringView view = batch->field(0).strings(session); + REQUIRE(view.size() == 2); + REQUIRE(view[0] == "inlined"sv); + REQUIRE(view[1] == long1); +} + +TEST_CASE("strings() on a non-utf8 array", "[strings]") { + Session session; + std::vector data = {1}; + Array a = Array::primitive(data); + REQUIRE_THROWS_AS(a.strings(session), VortexException); + REQUIRE_THROWS_AS(a.bytes(session), VortexException); +} +} // namespace diff --git a/lang/cpp/tests/writer.cpp b/lang/cpp/tests/writer.cpp new file mode 100644 index 00000000000..dda8d8c51d0 --- /dev/null +++ b/lang/cpp/tests/writer.cpp @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include + +#include "common.hpp" +#include "vortex/dtype.hpp" +#include "vortex/error.hpp" +#include "vortex/session.hpp" + +using namespace vortex; +using vortex_test::TempPath; + +TEST_CASE("Dangling session writer", "[writer]") { + TempPath path = TempPath::unique(); + + { + Writer writer = Writer::open(Session {}, path.string(), vortex_test::sample_dtype()); + writer.push(vortex_test::sample_array()); + } + + REQUIRE_THROWS_AS(DataSource::open(Session {}, {path.string()}), VortexException); +} + +TEST_CASE("Unfinished writer", "[writer]") { + Session session; + TempPath path = TempPath::unique(); + + { + Writer writer = Writer::open(session, path.string(), vortex_test::sample_dtype()); + writer.push(vortex_test::sample_array()); + } + + REQUIRE_THROWS_AS(DataSource::open(session, {path.string()}), VortexException); +} + +TEST_CASE("Write finish() called twice", "[writer]") { + Session session; + TempPath path = TempPath::unique(); + + Writer writer = Writer::open(session, path.string(), vortex_test::sample_dtype()); + writer.push(vortex_test::sample_array()); + writer.finish(); + REQUIRE_THROWS_AS(writer.finish(), VortexException); + REQUIRE_THROWS_AS(writer.push(vortex_test::sample_array()), VortexException); +} + +TEST_CASE("Writer push with invalid dtype", "[writer]") { + Session session; + TempPath path = TempPath::unique(); + Writer writer = Writer::open(session, path.string(), dtype::null()); + REQUIRE_THROWS_AS(writer.push(vortex_test::sample_array()), VortexException); + REQUIRE_THROWS_AS(writer.push(vortex_test::sample_array()), VortexException); +} From 50419b22f8c5b476c624f8eb0b4762510d5dc7a2 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Tue, 14 Jul 2026 17:40:35 +0100 Subject: [PATCH 2/3] initial --- lang/cpp/examples/.gitignore | 1 + lang/cpp/examples/CMakeLists.txt | 7 ++ lang/cpp/examples/dtype.cpp | 109 ++++++++++++++++++++++++++++ lang/cpp/examples/reader.cpp | 33 +++++++++ lang/cpp/examples/scan.cpp | 106 +++++++++++++++++++++++++++ lang/cpp/examples/scan_to_arrow.cpp | 65 +++++++++++++++++ lang/cpp/examples/writer.cpp | 54 ++++++++++++++ 7 files changed, 375 insertions(+) create mode 100644 lang/cpp/examples/.gitignore create mode 100644 lang/cpp/examples/CMakeLists.txt create mode 100644 lang/cpp/examples/dtype.cpp create mode 100644 lang/cpp/examples/reader.cpp create mode 100644 lang/cpp/examples/scan.cpp create mode 100644 lang/cpp/examples/scan_to_arrow.cpp create mode 100644 lang/cpp/examples/writer.cpp diff --git a/lang/cpp/examples/.gitignore b/lang/cpp/examples/.gitignore new file mode 100644 index 00000000000..065a36090f7 --- /dev/null +++ b/lang/cpp/examples/.gitignore @@ -0,0 +1 @@ +!goldenfiles/example.parquet \ No newline at end of file diff --git a/lang/cpp/examples/CMakeLists.txt b/lang/cpp/examples/CMakeLists.txt new file mode 100644 index 00000000000..2733b745b96 --- /dev/null +++ b/lang/cpp/examples/CMakeLists.txt @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +foreach(name reader writer dtype scan scan_to_arrow) + add_executable(${name} ${name}.cpp) + target_link_libraries(${name} PRIVATE vortex_cxx_shared nanoarrow_shared) +endforeach() diff --git a/lang/cpp/examples/dtype.cpp b/lang/cpp/examples/dtype.cpp new file mode 100644 index 00000000000..bdaaab1b3df --- /dev/null +++ b/lang/cpp/examples/dtype.cpp @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include +#include +#include + +using vortex::DataSource; +using vortex::DataType; +using vortex::DataTypeVariant; +using vortex::Field; +using vortex::Session; +using vortex::VortexException; + +static void print_dtype(const DataType &dtype); + +static std::string_view ptype_name(vortex::PType p) { + using enum vortex::PType; + switch (p) { + case U8: + return "uint8_t"; + case U16: + return "uint16_t"; + case U32: + return "uint32_t"; + case U64: + return "uint64_t"; + case I8: + return "int8_t"; + case I16: + return "int16_t"; + case I32: + return "int32_t"; + case I64: + return "int64_t"; + case F16: + return "float16"; + case F32: + return "float"; + case F64: + return "double"; + } + return "?"; +} + +static void print_struct(const DataType &dtype) { + std::cout << "struct(\n"; + for (const Field &field : dtype.fields()) { + std::cout << " " << field.name << " = "; + print_dtype(field.dtype); + } + std::cout << ")"; +} + +static void print_dtype(const DataType &dtype) { + using enum DataTypeVariant; + switch (dtype.variant()) { + case Null: + std::cout << "null"; + break; + case Bool: + std::cout << "bool"; + break; + case Utf8: + std::cout << "utf8"; + break; + case Binary: + std::cout << "binary"; + break; + case Extension: + std::cout << "extension"; + break; + case Primitive: + std::cout << "primitive(" << ptype_name(dtype.primitive_type()) << ")"; + break; + case Struct: + print_struct(dtype); + break; + case List: + std::cout << "list("; + print_dtype(dtype.list_element()); + std::cout << ")"; + break; + case FixedSizeList: + std::cout << "fixed_list("; + print_dtype(dtype.fixed_size_list_element()); + std::cout << ")"; + break; + case Decimal: + std::cout << "decimal(precision=" << static_cast(dtype.decimal_precision()) + << ", scale=" << static_cast(dtype.decimal_scale()) << ")"; + break; + } + std::cout << (dtype.nullable() ? '?' : ' ') << '\n'; +} + +int main(int argc, char **argv) { + if (argc != 2) { + std::cerr << "Usage: dtype \n"; + return 1; + } + + Session session; + DataSource ds = DataSource::open(session, {argv[1]}); + DataType dt = ds.dtype(); + std::cout << "dtype: "; + print_dtype(dt); + return 0; +} diff --git a/lang/cpp/examples/reader.cpp b/lang/cpp/examples/reader.cpp new file mode 100644 index 00000000000..48656c046f7 --- /dev/null +++ b/lang/cpp/examples/reader.cpp @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include + +#include +#include + +using namespace std::string_view_literals; +using namespace vortex; +using namespace expr; +using namespace ops; // overloaded >= for Expressions +namespace fs = std::filesystem; + +int main() { + const Session session; + const DataSource ds = DataSource::open(session, {"people*.vortex", "me.vortex"}); + Scan scan = ds.scan({.filter = col("height") >= lit(50)}); + + for (Partition &partition : scan.partitions()) { + for (Array &array : partition.batches()) { + const Array age = array.field("age"); + const PrimitiveView age_view = age.values(session); + const std::span age_values = age_view.values(); + for (uint8_t value : age_values) { + std::cout << int(value) << " "; + } + } + } + std::cout << "\n"; + + return 0; +} diff --git a/lang/cpp/examples/scan.cpp b/lang/cpp/examples/scan.cpp new file mode 100644 index 00000000000..1a9711888ec --- /dev/null +++ b/lang/cpp/examples/scan.cpp @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include + +#include +#include +#include +#include + +#include +#include + +using vortex::DataSource; +using vortex::Estimate; +using vortex::Scan; +using vortex::Session; + +static void print_estimate(const char *what, const Estimate &est) { + using enum vortex::EstimateType; + switch (est.type()) { + case Unknown: + std::cout << what << ": unknown\n"; + break; + case Exact: + std::cout << what << ": " << est.value() << '\n'; + break; + case Inexact: + std::cout << what << ": at most " << est.value() << '\n'; + break; + } +} + +struct ScanStats { + size_t partitions = 0; + size_t arrays = 0; + size_t rows = 0; +}; + +static ScanStats worker(Scan &scan) { + ScanStats stats; + while (auto partition = scan.next_partition()) { + ++stats.partitions; + while (auto array = partition->next()) { + ++stats.arrays; + stats.rows += array->size(); + } + } + return stats; +} + +int main(int argc, char **argv) { + size_t num_threads = 0; + int opt = 0; + while ((opt = getopt(argc, argv, "j:")) != -1) { + switch (opt) { + case 'j': + num_threads = static_cast(std::atoi(optarg)); + break; + default: + std::cerr << "Multi-threaded file scan\nUsage: scan [-j " + "threads] \n"; + return 1; + } + } + if (optind + 1 != argc) { + std::cerr << "Multi-threaded file scan\nUsage: scan [-j threads] \n"; + return 1; + } + + const Session session; + std::cout << "Opening files: " << argv[optind] << '\n'; + const DataSource ds = DataSource::open(session, {argv[optind]}); + + print_estimate("Data source row count", ds.row_count()); + + Scan scan = ds.scan(); + print_estimate("Partition count", scan.partition_count()); + + if (num_threads == 0) { + num_threads = scan.partition_count().value_or(1); + } + + std::cout << "Starting scan, using " << num_threads << " threads\n"; + std::vector threads; + threads.reserve(num_threads); + std::vector results(num_threads); + + for (size_t i = 0; i < num_threads; ++i) { + threads.emplace_back([i, &scan, &results] { results[i] = worker(scan); }); + } + for (auto &t : threads) { + t.join(); + } + + ScanStats total; + for (const auto &r : results) { + total.partitions += r.partitions; + total.arrays += r.arrays; + total.rows += r.rows; + } + std::cout << "Finished scan, processed " << total.partitions << " partitions, " << total.arrays + << " arrays, " << total.rows << " rows\n"; + return 0; +} diff --git a/lang/cpp/examples/scan_to_arrow.cpp b/lang/cpp/examples/scan_to_arrow.cpp new file mode 100644 index 00000000000..30369e24d1b --- /dev/null +++ b/lang/cpp/examples/scan_to_arrow.cpp @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include + +typedef struct ArrowSchema FFI_ArrowSchema; +typedef struct ArrowArray FFI_ArrowArray; +typedef struct ArrowArrayStream FFI_ArrowArrayStream; +#define USE_OWN_ARROW 1 + +#include +#include +#include +#include + +using vortex::ArrowStream; +using vortex::DataSource; +using vortex::DataType; +using vortex::Scan; +using vortex::Session; +using vortex::VortexException; + +int main(int argc, char **argv) { + if (argc != 2) { + std::cerr << "Scan vortex files to Arrow\nUsage: scan_to_arrow \n"; + return 1; + } + const char *paths = argv[1]; + + Session session; + DataSource ds = DataSource::open(session, {paths}); + Scan scan = ds.scan(); + + DataType out_dtype = ds.dtype(); + ArrowSchema schema = out_dtype.to_arrow(); + char schema_buf[10 * 1024]; + const int64_t schema_len = ArrowSchemaToString(&schema, schema_buf, sizeof schema_buf, 1); + std::cout << "arrow schema: " << std::string_view {schema_buf, static_cast(schema_len)} << '\n'; + if (schema.release != nullptr) { + schema.release(&schema); + } + + ArrowError arrow_error; + ArrowErrorInit(&arrow_error); + + size_t partition_idx = 0; + while (auto partition = scan.next_partition()) { + ArrowStream stream = std::move(*partition).into_arrow_stream(); + + size_t rows = 0; + size_t array_count = 0; + ArrowArray array = {}; + while (ArrowArrayStreamGetNext(stream.raw(), &array, &arrow_error) == NANOARROW_OK && + array.release != nullptr) { + rows += array.length; + ++array_count; + array.release(&array); + std::memset(&array, 0, sizeof(array)); + } + std::cout << "Read partition " << partition_idx << " to Arrow, " << array_count << " arrays, " << rows + << " rows\n"; + ++partition_idx; + } + return 0; +} diff --git a/lang/cpp/examples/writer.cpp b/lang/cpp/examples/writer.cpp new file mode 100644 index 00000000000..c16a6c7718f --- /dev/null +++ b/lang/cpp/examples/writer.cpp @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include +#include + +#include +#include + +using namespace vortex; +using namespace expr::ops; +using dtype::Nullable; + +int main(int argc, char **argv) { + if (argc != 2) { + return 1; + } + + const Session session; + const DataType dtype = dtype::struct_({ + {"age", dtype::uint8()}, + {"height", dtype::uint16(Nullable)}, + }); + + constexpr size_t SAMPLE_ROWS = 100; + std::vector age_buffer(SAMPLE_ROWS); + std::vector height_buffer(SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + age_buffer[i] = static_cast(i); + height_buffer[i] = static_cast((i + 1) % 200); + } + + Array age = Array::primitive(age_buffer); + Array array = make_struct({ + {"age", age}, + {"height", Array::primitive(height_buffer, AllValid)}, + }); + + Expression age_gt_10 = expr::col("age") > expr::lit(10); + Array validity_array = array.apply(age_gt_10); + + const Validity validity {ValidityType::Array, validity_array}; + Array array2 = make_struct({ + {"age", age}, + {"height", Array::primitive(height_buffer, validity)}, + }); + + Writer writer = Writer::open(session, argv[1], dtype); + writer.push(array); + writer.push(array2); + writer.finish(); + + return 0; +} From 8d8da4274237d19f3cc289b41c1875c6045607a0 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Tue, 14 Jul 2026 17:51:33 +0100 Subject: [PATCH 3/3] examples Signed-off-by: Mikhail Kot --- .github/workflows/ci.yml | 20 +++++++++++++++++--- lang/cpp/CMakeLists.txt | 8 +++++++- lang/cpp/examples/dtype.cpp | 8 +++----- lang/cpp/examples/writer.cpp | 4 +++- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc41ec4d2bd..8701b04f87f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -550,14 +550,28 @@ jobs: cargo +$NIGHTLY_TOOLCHAIN build --locked --no-default-features \ --target x86_64-unknown-linux-gnu -Zbuild-std \ -p vortex-ffi - - name: Build C++ library (asan) + - name: Build C++ tests and examples (asan) run: | mkdir -p build - cmake -S lang/cpp -B build -DSANITIZER=asan -DTARGET_TRIPLE="x86_64-unknown-linux-gnu" + cmake -S lang/cpp -Bbuild -DSANITIZER=asan -DBUILD_TESTS=1 -DBUILD_EXAMPLES=1 -DTARGET_TRIPLE="x86_64-unknown-linux-gnu" cmake --build build --parallel $(nproc) - name: Run C++ tests run: | - lang/cpp/build/tests/vortex_cxx_test + build/tests/vortex_cxx_test + - name: Run C++ examples + run: | + cd build/examples + # filesystem loop detected + rm -fr ../_deps/nanoarrow-src/python/subprojects/arrow-nanoarrow + + ./writer people0.vortex + ./writer people1.vortex + ./writer me.vortex + + ./scan '*.vortex' + ./dtype me.vortex + + ./reader sqllogic-test: name: "SQL logic tests" diff --git a/lang/cpp/CMakeLists.txt b/lang/cpp/CMakeLists.txt index 8949c1977cc..235764c3987 100644 --- a/lang/cpp/CMakeLists.txt +++ b/lang/cpp/CMakeLists.txt @@ -125,14 +125,16 @@ if(TARGET vortex_ffi_shared) endif() endif() -if (BUILD_TESTS) +if (BUILD_TESTS OR BUILD_EXAMPLES) FetchContent_Declare( Nanoarrow GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow GIT_TAG apache-arrow-nanoarrow-0.8.0 ) FetchContent_MakeAvailable(Nanoarrow) +endif() +if (BUILD_TESTS) FetchContent_Declare( Catch GIT_REPOSITORY https://github.com/catchorg/Catch2.git @@ -145,3 +147,7 @@ if (BUILD_TESTS) include(CTest) add_subdirectory(tests) endif() + +if (BUILD_EXAMPLES) + add_subdirectory(examples) +endif() diff --git a/lang/cpp/examples/dtype.cpp b/lang/cpp/examples/dtype.cpp index bdaaab1b3df..b70871ee4d1 100644 --- a/lang/cpp/examples/dtype.cpp +++ b/lang/cpp/examples/dtype.cpp @@ -8,9 +8,7 @@ using vortex::DataSource; using vortex::DataType; using vortex::DataTypeVariant; -using vortex::Field; using vortex::Session; -using vortex::VortexException; static void print_dtype(const DataType &dtype); @@ -45,9 +43,9 @@ static std::string_view ptype_name(vortex::PType p) { static void print_struct(const DataType &dtype) { std::cout << "struct(\n"; - for (const Field &field : dtype.fields()) { - std::cout << " " << field.name << " = "; - print_dtype(field.dtype); + for (const auto &[name, dtype] : dtype.fields()) { + std::cout << " " << name << " = "; + print_dtype(dtype); } std::cout << ")"; } diff --git a/lang/cpp/examples/writer.cpp b/lang/cpp/examples/writer.cpp index c16a6c7718f..5315b2a1c97 100644 --- a/lang/cpp/examples/writer.cpp +++ b/lang/cpp/examples/writer.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: CC-BY-4.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +#include "vortex/array.hpp" #include #include #include @@ -10,6 +11,7 @@ using namespace vortex; using namespace expr::ops; using dtype::Nullable; +using enum ValidityType; int main(int argc, char **argv) { if (argc != 2) { @@ -39,7 +41,7 @@ int main(int argc, char **argv) { Expression age_gt_10 = expr::col("age") > expr::lit(10); Array validity_array = array.apply(age_gt_10); - const Validity validity {ValidityType::Array, validity_array}; + const Validity validity = Validity::from_array(validity_array); Array array2 = make_struct({ {"age", age}, {"height", Array::primitive(height_buffer, validity)},