Skip to content

Scalar truncation is implemented over scalar values instead of scalars and can operate in place #6414

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
robert3005 merged 9 commits into from
Feb 16, 2026
Merged

Scalar truncation is implemented over scalar values instead of scalars and can operate in place #6414

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
eaa93f3
Scalar truncation is implemented over scalar values instead of scalar…
robert3005 Feb 10, 2026
eff67b0
comments
robert3005 Feb 11, 2026
e0e380a
api
robert3005 Feb 13, 2026
fc24e14
move
robert3005 Feb 16, 2026
f68db25
format
robert3005 Feb 16, 2026
34c69ce
comments
robert3005 Feb 16, 2026
1c6f4dd
eh
robert3005 Feb 16, 2026
9e476d3
lock
robert3005 Feb 16, 2026
dcae7e8
stats
robert3005 Feb 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 56 additions & 49 deletions vortex-cuda/src/layout.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@ use vortex_array::normalize::Operation;
use vortex_array::serde::ArrayParts;
use vortex_array::serde::SerializeOptions;
use vortex_array::session::ArrayRegistry;
use vortex_array::stats::StatsSetRef;
use vortex_buffer::Alignment;
use vortex_buffer::BufferString;
use vortex_buffer::ByteBuffer;
use vortex_dtype::DType;
use vortex_dtype::FieldMask;
Expand All @@ -54,15 +56,17 @@ use vortex_layout::LayoutRef;
use vortex_layout::LayoutStrategy;
use vortex_layout::VTable;
use vortex_layout::layouts::SharedArrayFuture;
use vortex_layout::layouts::zoned::lower_bound;
use vortex_layout::layouts::zoned::upper_bound;
use vortex_layout::segments::SegmentId;
use vortex_layout::segments::SegmentSinkRef;
use vortex_layout::segments::SegmentSource;
use vortex_layout::sequence::SendableSequentialStream;
use vortex_layout::sequence::SequencePointer;
use vortex_layout::vtable;
use vortex_mask::Mask;
use vortex_scalar::Scalar;
use vortex_scalar::ScalarTruncation;
use vortex_scalar::lower_bound;
use vortex_scalar::upper_bound;
use vortex_session::VortexSession;
use vortex_utils::aliases::hash_map::HashMap;

Expand Down Expand Up @@ -456,6 +460,22 @@ impl CudaFlatLayoutStrategy {
}
}

fn truncate_scalar_stat<F: Fn(Scalar) -> Option<(Scalar, bool)>>(
statistics: StatsSetRef<'_>,
stat: Stat,
truncation: F,
) {
if let Some(sv) = statistics.get(stat) {
if let Some((truncated_value, truncated)) = truncation(sv.into_inner()) {
if truncated && let Some(v) = truncated_value.into_value() {
statistics.set(stat, Precision::Inexact(v));
}
} else {
statistics.clear(stat)
}
}
}

#[async_trait]
impl LayoutStrategy for CudaFlatLayoutStrategy {
async fn write_stream(
Expand All @@ -474,55 +494,42 @@ impl LayoutStrategy for CudaFlatLayoutStrategy {
let (sequence_id, chunk) = chunk?;
let row_count = chunk.len() as u64;

// Truncate variable-length statistics.
match chunk.dtype() {
DType::Utf8(_) => {
if let Some(sv) = chunk.statistics().get(Stat::Min) {
let (value, truncated) = lower_bound(
sv.into_inner().as_utf8(),
options.max_variable_length_statistics_size,
);
if truncated && let Some(v) = value.into_value() {
chunk.statistics().set(Stat::Min, Precision::Inexact(v));
}
}
if let Some(sv) = chunk.statistics().get(Stat::Max) {
let (value, truncated) = upper_bound(
sv.into_inner().as_utf8(),
options.max_variable_length_statistics_size,
);
if let Some(upper_bound) = value {
if truncated && let Some(v) = upper_bound.into_value() {
chunk.statistics().set(Stat::Max, Precision::Inexact(v));
}
} else {
chunk.statistics().clear(Stat::Max)
}
}
DType::Utf8(n) => {
truncate_scalar_stat(chunk.statistics(), Stat::Min, |v| {
lower_bound(
BufferString::from_scalar(v)
.vortex_expect("utf8 scalar must be a BufferString"),
self.max_variable_length_statistics_size,
*n,
)
});
truncate_scalar_stat(chunk.statistics(), Stat::Max, |v| {
upper_bound(
BufferString::from_scalar(v)
.vortex_expect("utf8 scalar must be a BufferString"),
self.max_variable_length_statistics_size,
*n,
)
});
}
DType::Binary(_) => {
if let Some(sv) = chunk.statistics().get(Stat::Min) {
let (value, truncated) = lower_bound(
sv.into_inner().as_binary(),
options.max_variable_length_statistics_size,
);
if truncated && let Some(v) = value.into_value() {
chunk.statistics().set(Stat::Min, Precision::Inexact(v));
}
}
if let Some(sv) = chunk.statistics().get(Stat::Max) {
let (value, truncated) = upper_bound(
sv.into_inner().as_binary(),
options.max_variable_length_statistics_size,
);
if let Some(upper_bound) = value {
if truncated && let Some(v) = upper_bound.into_value() {
chunk.statistics().set(Stat::Max, Precision::Inexact(v));
}
} else {
chunk.statistics().clear(Stat::Max)
}
}
DType::Binary(n) => {
truncate_scalar_stat(chunk.statistics(), Stat::Min, |v| {
lower_bound(
ByteBuffer::from_scalar(v)
.vortex_expect("binary scalar must be a ByteBuffer"),
self.max_variable_length_statistics_size,
*n,
)
});
truncate_scalar_stat(chunk.statistics(), Stat::Max, |v| {
upper_bound(
ByteBuffer::from_scalar(v)
.vortex_expect("binary scalar must be a ByteBuffer"),
self.max_variable_length_statistics_size,
*n,
)
});
}
_ => {}
}
Expand Down
4 changes: 0 additions & 4 deletions vortex-layout/public-api.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -960,10 +960,6 @@ pub const vortex_layout::layouts::zoned::MAX_IS_TRUNCATED: &str

pub const vortex_layout::layouts::zoned::MIN_IS_TRUNCATED: &str

pub fn vortex_layout::layouts::zoned::lower_bound(value: impl vortex_layout::layouts::zoned::builder::ScalarTruncation, max_length: usize) -> (vortex_scalar::scalar::Scalar, bool)

pub fn vortex_layout::layouts::zoned::upper_bound(value: impl vortex_layout::layouts::zoned::builder::ScalarTruncation, max_length: usize) -> (core::option::Option<vortex_scalar::scalar::Scalar>, bool)

pub type vortex_layout::layouts::SharedArrayFuture = futures_util::future::future::shared::Shared<futures_core::future::BoxFuture<'static, vortex_error::SharedVortexResult<vortex_array::array::ArrayRef>>>

pub mod vortex_layout::segments
Expand Down
113 changes: 60 additions & 53 deletions vortex-layout/src/layouts/flat/writer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,24 @@ use vortex_array::normalize::NormalizeOptions;
use vortex_array::normalize::Operation;
use vortex_array::serde::SerializeOptions;
use vortex_array::session::ArrayRegistry;
use vortex_array::stats::StatsSetRef;
use vortex_buffer::BufferString;
use vortex_buffer::ByteBuffer;
use vortex_dtype::DType;
use vortex_error::VortexExpect;
use vortex_error::VortexResult;
use vortex_error::vortex_bail;
use vortex_io::runtime::Handle;
use vortex_scalar::Scalar;
use vortex_scalar::ScalarTruncation;
use vortex_scalar::lower_bound;
use vortex_scalar::upper_bound;

use crate::IntoLayout;
use crate::LayoutRef;
use crate::LayoutStrategy;
use crate::layouts::flat::FlatLayout;
use crate::layouts::flat::flat_layout_inline_array_node;
use crate::layouts::zoned::lower_bound;
use crate::layouts::zoned::upper_bound;
use crate::segments::SegmentSinkRef;
use crate::sequence::SendableSequentialStream;
use crate::sequence::SequencePointer;
Expand Down Expand Up @@ -69,6 +75,22 @@ impl FlatLayoutStrategy {
}
}

fn truncate_scalar_stat<F: Fn(Scalar) -> Option<(Scalar, bool)>>(
statistics: StatsSetRef<'_>,
stat: Stat,
truncation: F,
) {
if let Some(sv) = statistics.get(stat) {
if let Some((truncated_value, truncated)) = truncation(sv.into_inner()) {
if truncated && let Some(v) = truncated_value.into_value() {
statistics.set(stat, Precision::Inexact(v));
}
} else {
statistics.clear(stat)
}
}
}

#[async_trait]
impl LayoutStrategy for FlatLayoutStrategy {
async fn write_stream(
Expand All @@ -80,7 +102,6 @@ impl LayoutStrategy for FlatLayoutStrategy {
_handle: Handle,
) -> VortexResult<LayoutRef> {
let ctx = ctx.clone();
let options = self.clone();
let Some(chunk) = stream.next().await else {
vortex_bail!("flat layout needs a single chunk");
};
Expand All @@ -89,60 +110,46 @@ impl LayoutStrategy for FlatLayoutStrategy {
let row_count = chunk.len() as u64;

match chunk.dtype() {
DType::Utf8(_) => {
if let Some(sv) = chunk.statistics().get(Stat::Min) {
let (value, truncated) = lower_bound(
sv.into_inner().as_utf8(),
options.max_variable_length_statistics_size,
);
if truncated && let Some(v) = value.into_value() {
chunk.statistics().set(Stat::Min, Precision::Inexact(v));
}
}

if let Some(sv) = chunk.statistics().get(Stat::Max) {
let (value, truncated) = upper_bound(
sv.into_inner().as_utf8(),
options.max_variable_length_statistics_size,
);
if let Some(upper_bound) = value {
if truncated && let Some(v) = upper_bound.into_value() {
chunk.statistics().set(Stat::Max, Precision::Inexact(v));
}
} else {
chunk.statistics().clear(Stat::Max)
}
}
DType::Utf8(n) => {
truncate_scalar_stat(chunk.statistics(), Stat::Min, |v| {
lower_bound(
BufferString::from_scalar(v)
.vortex_expect("utf8 scalar must be a BufferString"),
self.max_variable_length_statistics_size,
*n,
)
});
truncate_scalar_stat(chunk.statistics(), Stat::Max, |v| {
upper_bound(
BufferString::from_scalar(v)
.vortex_expect("utf8 scalar must be a BufferString"),
self.max_variable_length_statistics_size,
*n,
)
});
}
DType::Binary(_) => {
if let Some(sv) = chunk.statistics().get(Stat::Min) {
let (value, truncated) = lower_bound(
sv.into_inner().as_binary(),
options.max_variable_length_statistics_size,
);
if truncated && let Some(v) = value.into_value() {
chunk.statistics().set(Stat::Min, Precision::Inexact(v));
}
}

if let Some(sv) = chunk.statistics().get(Stat::Max) {
let (value, truncated) = upper_bound(
sv.into_inner().as_binary(),
options.max_variable_length_statistics_size,
);
if let Some(upper_bound) = value {
if truncated && let Some(v) = upper_bound.into_value() {
chunk.statistics().set(Stat::Max, Precision::Inexact(v));
}
} else {
chunk.statistics().clear(Stat::Max)
}
}
DType::Binary(n) => {
truncate_scalar_stat(chunk.statistics(), Stat::Min, |v| {
lower_bound(
ByteBuffer::from_scalar(v)
.vortex_expect("binary scalar must be a ByteBuffer"),
self.max_variable_length_statistics_size,
*n,
)
});
truncate_scalar_stat(chunk.statistics(), Stat::Max, |v| {
upper_bound(
ByteBuffer::from_scalar(v)
.vortex_expect("binary scalar must be a ByteBuffer"),
self.max_variable_length_statistics_size,
*n,
)
});
}
_ => {}
}

let chunk = if let Some(allowed) = &options.allowed_encodings {
let chunk = if let Some(allowed) = &self.allowed_encodings {
chunk.normalize(&mut NormalizeOptions {
allowed,
operation: Operation::Error,
Expand All @@ -155,7 +162,7 @@ impl LayoutStrategy for FlatLayoutStrategy {
&ctx,
&SerializeOptions {
offset: 0,
include_padding: options.include_padding,
include_padding: self.include_padding,
},
)?;
// there is at least the flatbuffer and the length
Expand Down
Loading
Loading