From d1556ec62ac965f470c1836271e204a47acaefc9 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Fri, 4 Sep 2026 15:17:01 +0100 Subject: [PATCH 1/6] initial Signed-off-by: Mikhail Kot --- vortex-array/src/arrays/filter/kernel.rs | 3 + vortex-array/src/arrays/scalar_fn/rules.rs | 64 ++++++++++++++++++++++ vortex-array/src/executor.rs | 2 +- 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/vortex-array/src/arrays/filter/kernel.rs b/vortex-array/src/arrays/filter/kernel.rs index a62dc7910a0..ea86f5ecac0 100644 --- a/vortex-array/src/arrays/filter/kernel.rs +++ b/vortex-array/src/arrays/filter/kernel.rs @@ -26,10 +26,13 @@ use crate::kernel::ExecuteParentKernel; use crate::matcher::Matcher; use crate::optimizer::kernels::ArrayKernelsExt; use crate::optimizer::rules::ArrayParentReduceRule; +use crate::scalar_fn::ScalarFnPlugin; +use crate::scalar_fn::fns::binary::Binary; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); kernels.register_execute_parent_kernel(Dict.id(), Filter, TakeExecuteAdaptor(Filter)); + kernels.register_execute_parent_kernel(Binary.id(), Filter, FilterScalarFnUnaryPushDownRule); } pub trait FilterReduce: VTable { diff --git a/vortex-array/src/arrays/scalar_fn/rules.rs b/vortex-array/src/arrays/scalar_fn/rules.rs index 52373d80a33..10ead1f1029 100644 --- a/vortex-array/src/arrays/scalar_fn/rules.rs +++ b/vortex-array/src/arrays/scalar_fn/rules.rs @@ -5,16 +5,20 @@ use itertools::Itertools; use vortex_error::VortexResult; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Filter; +use crate::arrays::FilterArray; use crate::arrays::ScalarFn; use crate::arrays::ScalarFnArray; use crate::arrays::Slice; use crate::arrays::StructArray; use crate::arrays::scalar_fn::ScalarFnArrayExt; +use crate::kernel::ExecuteParentKernel; +use crate::optimizer::kernels::execute_parent_key; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ArrayReduceRule; use crate::optimizer::rules::ParentRuleSet; @@ -134,6 +138,66 @@ impl ArrayParentReduceRule for ScalarFnUnaryFilterPushDownRule { } } +#[derive(Debug)] +struct FilterScalarFnUnaryPushDownRule; + +impl ExecuteParentKernel for FilterScalarFnUnaryPushDownRule { + type Parent = ScalarFn; + + fn execute_parent( + &self, + child: ArrayView<'_, Filter>, + parent: ArrayView<'_, ScalarFn>, + _child_idx: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + // If we have one non-constant child, and ScalarFn has a registered + // kernel for given encoding id (e.g. can operate on compressed data), + // it's faster to pull Filter up so that it doesn't canonicalize its + // child. + // + // Fn(Filter(x), consts) -> Filter(Fn(x, consts)) + // + // Example: clickbench q1, SELECT * FROM hits WHERE AdvEngineID <> 0; + // AdvEngineID <> 0 is Binary over Sparse, but FlatReader applies + // Filter over Sparse, so we get Binary(Filter(Sparse)). Filter(Sparse) + // canonicalizes. + let mut non_const_child_id: usize = usize::MAX; + for (i, child) in parent.iter_children().enumerate() { + if child.is::() { + continue; + } + if non_const_child_id != usize::MAX { + return Ok(None); + } + non_const_child_id = i; + } + + let new_non_const_grandchild = parent.child_at(non_const_child_id); + + let key = execute_parent_key( + parent.scalar_fn.id(), + new_non_const_grandchild.encoding_id(), + ); + if !ctx.execute_parent_kernels.contains_key(&key) { + return Ok(None); + }; + + let new_grandchildren: Vec<_> = parent + .iter_children() + .map(|child| match child.as_constant() { + Some(scalar) => ConstantArray::new(scalar, parent.len()).into_array(), + None => new_non_const_grandchild.clone(), + }) + .collect(); + + let new_child = ScalarFnArray::try_new(parent.scalar_fn().clone(), new_grandchildren)?; + let mask = child.filter_mask().clone(); + let new_parent = FilterArray::try_new(new_child.into_array(), mask)?; + Ok(Some(new_parent.into_array())) + } +} + #[cfg(test)] mod tests { use vortex_error::VortexExpect; diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index dab9ccfcfb5..4358202c82c 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -353,7 +353,7 @@ pub struct ExecutionCtx { session: VortexSession, // OnceLock avoids cloning the session allocator when a context does not allocate. allocator: OnceLock, - execute_parent_kernels: Arc, + pub(crate) execute_parent_kernels: Arc, #[cfg(debug_assertions)] id: usize, #[cfg(debug_assertions)] From 5a0e7a561c3cf8ea19cbf7e42a63fdde7cd52fd1 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Fri, 4 Sep 2026 16:03:54 +0100 Subject: [PATCH 2/6] better Signed-off-by: Mikhail Kot --- vortex-array/src/arrays/filter/kernel.rs | 86 ++++++++++++++++++++++ vortex-array/src/arrays/scalar_fn/rules.rs | 64 ---------------- vortex-array/src/executor.rs | 2 +- 3 files changed, 87 insertions(+), 65 deletions(-) diff --git a/vortex-array/src/arrays/filter/kernel.rs b/vortex-array/src/arrays/filter/kernel.rs index ea86f5ecac0..7fbae111304 100644 --- a/vortex-array/src/arrays/filter/kernel.rs +++ b/vortex-array/src/arrays/filter/kernel.rs @@ -8,6 +8,7 @@ //! [`FilterExecuteAdaptor`] bridge these into the execution model as //! [`ArrayParentReduceRule`] and [`ExecuteParentKernel`] respectively. +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_mask::Mask; use vortex_session::VortexSession; @@ -19,9 +20,17 @@ use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::array::VTable; +use crate::arrays::Constant; +use crate::arrays::ConstantArray; use crate::arrays::Dict; use crate::arrays::Filter; +use crate::arrays::FilterArray; +use crate::arrays::ScalarFn; +use crate::arrays::ScalarFnArray; use crate::arrays::dict::TakeExecuteAdaptor; +use crate::arrays::filter::FilterSlots; +use crate::arrays::scalar_fn::ScalarFnArrayExt; +use crate::execute_parent_for_child; use crate::kernel::ExecuteParentKernel; use crate::matcher::Matcher; use crate::optimizer::kernels::ArrayKernelsExt; @@ -138,3 +147,80 @@ where ::filter(array, parent.filter_mask(), ctx) } } + +#[derive(Debug)] +struct FilterScalarFnUnaryPushDownRule; + +impl ExecuteParentKernel for FilterScalarFnUnaryPushDownRule { + type Parent = ScalarFn; + + fn execute_parent( + &self, + child: ArrayView<'_, Filter>, + parent: ArrayView<'_, ScalarFn>, + child_idx: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + // If we have one non-constant child, and ScalarFn has a registered + // kernel for given encoding id (e.g. can operate on compressed data), + // it's faster to pull Filter up so that it doesn't canonicalize its + // child. + // + // Fn(Filter(x), consts) -> Filter(Fn(x, consts)) + // + // Example: clickbench q1, SELECT * FROM hits WHERE AdvEngineID <> 0; + // AdvEngineID <> 0 is Binary over Sparse, but FlatReader applies + // Filter over Sparse, so we get Binary(Filter(Sparse)). Filter(Sparse) + // canonicalizes. + if parent + .iter_children() + .filter(|c| !c.is::()) + .count() + != 1 + { + return Ok(None); + } + + // "x" in above formula + let new_non_const_grandchild = child.slots()[FilterSlots::CHILD] + .as_ref() + .vortex_expect("no child for Filter"); + + let unfiltered_len = new_non_const_grandchild.len(); + // (x, consts) + let new_grandchildren: Vec<_> = parent + .iter_children() + .map(|c| match c.as_constant() { + Some(scalar) => ConstantArray::new(scalar, unfiltered_len).into_array(), + // by above check this is the only non-const argument + None => new_non_const_grandchild.clone(), + }) + .collect(); + + // Fn(x, consts) + let new_child = + ScalarFnArray::try_new(parent.scalar_fn().clone(), new_grandchildren)?.into_array(); + + // Eagerly execute swapped Fn to avoid infinite runtime with + // ScalarFnUnaryFilterPushDownRule. + // + // This is Res = Fn(x, consts) + let new_child = execute_parent_for_child( + "filter_scalar_fn_pushdown", + &new_child, // parent, Fn + new_non_const_grandchild, // child, x + child_idx, + &ctx.execute_parent_kernels.clone(), + ctx, + )?; + let Some(new_child) = new_child else { + // All child kernels rejected, can't proceed + return Ok(None); + }; + + let mask = child.filter_mask().clone(); + // Filter(Res) + let new_parent = FilterArray::try_new(new_child.clone(), mask)?; + Ok(Some(new_parent.into_array())) + } +} diff --git a/vortex-array/src/arrays/scalar_fn/rules.rs b/vortex-array/src/arrays/scalar_fn/rules.rs index 10ead1f1029..52373d80a33 100644 --- a/vortex-array/src/arrays/scalar_fn/rules.rs +++ b/vortex-array/src/arrays/scalar_fn/rules.rs @@ -5,20 +5,16 @@ use itertools::Itertools; use vortex_error::VortexResult; use crate::ArrayRef; -use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Filter; -use crate::arrays::FilterArray; use crate::arrays::ScalarFn; use crate::arrays::ScalarFnArray; use crate::arrays::Slice; use crate::arrays::StructArray; use crate::arrays::scalar_fn::ScalarFnArrayExt; -use crate::kernel::ExecuteParentKernel; -use crate::optimizer::kernels::execute_parent_key; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ArrayReduceRule; use crate::optimizer::rules::ParentRuleSet; @@ -138,66 +134,6 @@ impl ArrayParentReduceRule for ScalarFnUnaryFilterPushDownRule { } } -#[derive(Debug)] -struct FilterScalarFnUnaryPushDownRule; - -impl ExecuteParentKernel for FilterScalarFnUnaryPushDownRule { - type Parent = ScalarFn; - - fn execute_parent( - &self, - child: ArrayView<'_, Filter>, - parent: ArrayView<'_, ScalarFn>, - _child_idx: usize, - ctx: &mut ExecutionCtx, - ) -> VortexResult> { - // If we have one non-constant child, and ScalarFn has a registered - // kernel for given encoding id (e.g. can operate on compressed data), - // it's faster to pull Filter up so that it doesn't canonicalize its - // child. - // - // Fn(Filter(x), consts) -> Filter(Fn(x, consts)) - // - // Example: clickbench q1, SELECT * FROM hits WHERE AdvEngineID <> 0; - // AdvEngineID <> 0 is Binary over Sparse, but FlatReader applies - // Filter over Sparse, so we get Binary(Filter(Sparse)). Filter(Sparse) - // canonicalizes. - let mut non_const_child_id: usize = usize::MAX; - for (i, child) in parent.iter_children().enumerate() { - if child.is::() { - continue; - } - if non_const_child_id != usize::MAX { - return Ok(None); - } - non_const_child_id = i; - } - - let new_non_const_grandchild = parent.child_at(non_const_child_id); - - let key = execute_parent_key( - parent.scalar_fn.id(), - new_non_const_grandchild.encoding_id(), - ); - if !ctx.execute_parent_kernels.contains_key(&key) { - return Ok(None); - }; - - let new_grandchildren: Vec<_> = parent - .iter_children() - .map(|child| match child.as_constant() { - Some(scalar) => ConstantArray::new(scalar, parent.len()).into_array(), - None => new_non_const_grandchild.clone(), - }) - .collect(); - - let new_child = ScalarFnArray::try_new(parent.scalar_fn().clone(), new_grandchildren)?; - let mask = child.filter_mask().clone(); - let new_parent = FilterArray::try_new(new_child.into_array(), mask)?; - Ok(Some(new_parent.into_array())) - } -} - #[cfg(test)] mod tests { use vortex_error::VortexExpect; diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index 4358202c82c..573716239ef 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -621,7 +621,7 @@ fn finalize_done( Ok((output, None)) } -fn execute_parent_for_child( +pub(crate) fn execute_parent_for_child( _phase: &'static str, parent: &ArrayRef, child: &ArrayRef, From c21203bd840fa59cae740f37b324df150b3651d8 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Fri, 4 Sep 2026 16:40:18 +0100 Subject: [PATCH 3/6] also register for Slice Signed-off-by: Mikhail Kot --- vortex-array/src/arrays/filter/kernel.rs | 160 ++++++++++++++--------- 1 file changed, 100 insertions(+), 60 deletions(-) diff --git a/vortex-array/src/arrays/filter/kernel.rs b/vortex-array/src/arrays/filter/kernel.rs index 7fbae111304..74ec0615746 100644 --- a/vortex-array/src/arrays/filter/kernel.rs +++ b/vortex-array/src/arrays/filter/kernel.rs @@ -27,21 +27,34 @@ use crate::arrays::Filter; use crate::arrays::FilterArray; use crate::arrays::ScalarFn; use crate::arrays::ScalarFnArray; +use crate::arrays::Slice; +use crate::arrays::SliceArray; use crate::arrays::dict::TakeExecuteAdaptor; use crate::arrays::filter::FilterSlots; use crate::arrays::scalar_fn::ScalarFnArrayExt; +use crate::arrays::slice::SliceSlots; use crate::execute_parent_for_child; use crate::kernel::ExecuteParentKernel; use crate::matcher::Matcher; use crate::optimizer::kernels::ArrayKernelsExt; use crate::optimizer::rules::ArrayParentReduceRule; use crate::scalar_fn::ScalarFnPlugin; +use crate::scalar_fn::fns::between::Between; use crate::scalar_fn::fns::binary::Binary; +use crate::scalar_fn::fns::fill_null::FillNull; pub(crate) fn initialize(session: &VortexSession) { let kernels = session.kernels(); kernels.register_execute_parent_kernel(Dict.id(), Filter, TakeExecuteAdaptor(Filter)); - kernels.register_execute_parent_kernel(Binary.id(), Filter, FilterScalarFnUnaryPushDownRule); + + for parent in [Binary.id(), Between.id(), FillNull.id()] { + kernels.register_execute_parent_kernel( + parent, + Filter, + FilterSliceScalarFnUnaryPushDownRule, + ); + kernels.register_execute_parent_kernel(parent, Slice, FilterSliceScalarFnUnaryPushDownRule); + } } pub trait FilterReduce: VTable { @@ -148,12 +161,71 @@ where } } +/// If we have one non-constant child, and ScalarFn has a registered +/// kernel for given encoding id (e.g. can operate on compressed data), +/// it's faster to pull V up so that it doesn't canonicalize its +/// child. +/// +/// Fn(V(x), consts) -> V(Fn(x, consts)) +/// +/// Returns new child, i.e. Fn(x, consts) with Fn applied eagerly. +/// +/// Example: clickbench q1, SELECT * FROM hits WHERE AdvEngineID <> 0; +/// AdvEngineID <> 0 is Binary over Sparse, but FlatReader applies +/// Filter over Sparse, so we get Binary(Filter(Sparse)). Filter(Sparse) +/// canonicalizes. +fn filter_slice_execute_parent( + child: ArrayView<'_, V>, + parent: ArrayView<'_, ScalarFn>, + child_idx: usize, + ctx: &mut ExecutionCtx, +) -> VortexResult> { + if parent + .iter_children() + .filter(|c| !c.is::()) + .count() + != 1 + { + return Ok(None); + } + + // "x" in above formula + let new_non_const_grandchild = child.slots()[CHILD_SLOT] + .as_ref() + .vortex_expect("no child for V"); + + let unfiltered_len = new_non_const_grandchild.len(); + // (x, consts) + let new_grandchildren: Vec<_> = parent + .iter_children() + .map(|c| match c.as_constant() { + Some(scalar) => ConstantArray::new(scalar, unfiltered_len).into_array(), + // by above check this is the only non-const argument + None => new_non_const_grandchild.clone(), + }) + .collect(); + + // Fn(x, consts) + let new_child = + ScalarFnArray::try_new(parent.scalar_fn().clone(), new_grandchildren)?.into_array(); + + // Eagerly execute swapped Fn to avoid infinite runtime with + // ScalarFnUnaryFilterPushDownRule. Returns Fn(x, consts) + execute_parent_for_child( + "filter_scalar_fn_pushdown", + &new_child, // parent, Fn + new_non_const_grandchild, // child, x + child_idx, + &ctx.execute_parent_kernels.clone(), + ctx, + ) +} + #[derive(Debug)] -struct FilterScalarFnUnaryPushDownRule; +struct FilterSliceScalarFnUnaryPushDownRule; -impl ExecuteParentKernel for FilterScalarFnUnaryPushDownRule { +impl ExecuteParentKernel for FilterSliceScalarFnUnaryPushDownRule { type Parent = ScalarFn; - fn execute_parent( &self, child: ArrayView<'_, Filter>, @@ -161,66 +233,34 @@ impl ExecuteParentKernel for FilterScalarFnUnaryPushDownRule { child_idx: usize, ctx: &mut ExecutionCtx, ) -> VortexResult> { - // If we have one non-constant child, and ScalarFn has a registered - // kernel for given encoding id (e.g. can operate on compressed data), - // it's faster to pull Filter up so that it doesn't canonicalize its - // child. - // - // Fn(Filter(x), consts) -> Filter(Fn(x, consts)) - // - // Example: clickbench q1, SELECT * FROM hits WHERE AdvEngineID <> 0; - // AdvEngineID <> 0 is Binary over Sparse, but FlatReader applies - // Filter over Sparse, so we get Binary(Filter(Sparse)). Filter(Sparse) - // canonicalizes. - if parent - .iter_children() - .filter(|c| !c.is::()) - .count() - != 1 - { - return Ok(None); - } - - // "x" in above formula - let new_non_const_grandchild = child.slots()[FilterSlots::CHILD] - .as_ref() - .vortex_expect("no child for Filter"); - - let unfiltered_len = new_non_const_grandchild.len(); - // (x, consts) - let new_grandchildren: Vec<_> = parent - .iter_children() - .map(|c| match c.as_constant() { - Some(scalar) => ConstantArray::new(scalar, unfiltered_len).into_array(), - // by above check this is the only non-const argument - None => new_non_const_grandchild.clone(), - }) - .collect(); - - // Fn(x, consts) - let new_child = - ScalarFnArray::try_new(parent.scalar_fn().clone(), new_grandchildren)?.into_array(); - - // Eagerly execute swapped Fn to avoid infinite runtime with - // ScalarFnUnaryFilterPushDownRule. - // - // This is Res = Fn(x, consts) - let new_child = execute_parent_for_child( - "filter_scalar_fn_pushdown", - &new_child, // parent, Fn - new_non_const_grandchild, // child, x - child_idx, - &ctx.execute_parent_kernels.clone(), - ctx, - )?; - let Some(new_child) = new_child else { - // All child kernels rejected, can't proceed + const SLOT: usize = FilterSlots::CHILD; + let Some(new_child) = + filter_slice_execute_parent::(child, parent, child_idx, ctx)? + else { return Ok(None); }; - let mask = child.filter_mask().clone(); - // Filter(Res) let new_parent = FilterArray::try_new(new_child.clone(), mask)?; Ok(Some(new_parent.into_array())) } } + +impl ExecuteParentKernel for FilterSliceScalarFnUnaryPushDownRule { + type Parent = ScalarFn; + fn execute_parent( + &self, + child: ArrayView<'_, Slice>, + parent: ArrayView<'_, ScalarFn>, + child_idx: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + const SLOT: usize = SliceSlots::CHILD; + let Some(new_child) = + filter_slice_execute_parent::(child, parent, child_idx, ctx)? + else { + return Ok(None); + }; + let new_parent = SliceArray::try_new(new_child.clone(), child.slice_range().clone())?; + Ok(Some(new_parent.into_array())) + } +} From 5e44d39bf2d0b6432d4de2e43c9b8ca59d4caf63 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Fri, 4 Sep 2026 17:10:18 +0100 Subject: [PATCH 4/6] apply slice eagerly Signed-off-by: Mikhail Kot --- vortex-array/src/arrays/filter/kernel.rs | 76 +++++++++++++----------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/vortex-array/src/arrays/filter/kernel.rs b/vortex-array/src/arrays/filter/kernel.rs index 74ec0615746..d4fc3f645bc 100644 --- a/vortex-array/src/arrays/filter/kernel.rs +++ b/vortex-array/src/arrays/filter/kernel.rs @@ -8,6 +8,8 @@ //! [`FilterExecuteAdaptor`] bridge these into the execution model as //! [`ArrayParentReduceRule`] and [`ExecuteParentKernel`] respectively. +use std::sync::Arc; + use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_mask::Mask; @@ -28,7 +30,6 @@ use crate::arrays::FilterArray; use crate::arrays::ScalarFn; use crate::arrays::ScalarFnArray; use crate::arrays::Slice; -use crate::arrays::SliceArray; use crate::arrays::dict::TakeExecuteAdaptor; use crate::arrays::filter::FilterSlots; use crate::arrays::scalar_fn::ScalarFnArrayExt; @@ -162,20 +163,18 @@ where } /// If we have one non-constant child, and ScalarFn has a registered -/// kernel for given encoding id (e.g. can operate on compressed data), -/// it's faster to pull V up so that it doesn't canonicalize its -/// child. -/// -/// Fn(V(x), consts) -> V(Fn(x, consts)) +/// kernel for the encoding of "inner" (e.g. can operate on compressed data), +/// it's faster to execute Fn on "inner" directly so that the we avoid +/// canonicalizing data while executing "inner" /// -/// Returns new child, i.e. Fn(x, consts) with Fn applied eagerly. +/// Returns Fn(inner, consts) with Fn applied eagerly. /// /// Example: clickbench q1, SELECT * FROM hits WHERE AdvEngineID <> 0; /// AdvEngineID <> 0 is Binary over Sparse, but FlatReader applies /// Filter over Sparse, so we get Binary(Filter(Sparse)). Filter(Sparse) /// canonicalizes. -fn filter_slice_execute_parent( - child: ArrayView<'_, V>, +fn scalar_fn_on_inner( + inner: &ArrayRef, parent: ArrayView<'_, ScalarFn>, child_idx: usize, ctx: &mut ExecutionCtx, @@ -189,34 +188,29 @@ fn filter_slice_execute_parent( return Ok(None); } - // "x" in above formula - let new_non_const_grandchild = child.slots()[CHILD_SLOT] - .as_ref() - .vortex_expect("no child for V"); - - let unfiltered_len = new_non_const_grandchild.len(); - // (x, consts) - let new_grandchildren: Vec<_> = parent + let inner_len = inner.len(); + // (inner, consts) + let new_children: Vec<_> = parent .iter_children() .map(|c| match c.as_constant() { - Some(scalar) => ConstantArray::new(scalar, unfiltered_len).into_array(), + Some(scalar) => ConstantArray::new(scalar, inner_len).into_array(), // by above check this is the only non-const argument - None => new_non_const_grandchild.clone(), + None => inner.clone(), }) .collect(); - // Fn(x, consts) - let new_child = - ScalarFnArray::try_new(parent.scalar_fn().clone(), new_grandchildren)?.into_array(); + // Fn(inner, consts) + let new_fn = ScalarFnArray::try_new(parent.scalar_fn().clone(), new_children)?.into_array(); // Eagerly execute swapped Fn to avoid infinite runtime with - // ScalarFnUnaryFilterPushDownRule. Returns Fn(x, consts) + // ScalarFnUnaryFilterPushDownRule. Returns Fn(inner, consts) + let kernels = Arc::clone(&ctx.execute_parent_kernels); execute_parent_for_child( "filter_scalar_fn_pushdown", - &new_child, // parent, Fn - new_non_const_grandchild, // child, x + &new_fn, // parent, Fn + inner, // child child_idx, - &ctx.execute_parent_kernels.clone(), + &kernels, ctx, ) } @@ -233,14 +227,14 @@ impl ExecuteParentKernel for FilterSliceScalarFnUnaryPushDownRule { child_idx: usize, ctx: &mut ExecutionCtx, ) -> VortexResult> { - const SLOT: usize = FilterSlots::CHILD; - let Some(new_child) = - filter_slice_execute_parent::(child, parent, child_idx, ctx)? - else { + let inner = child.slots()[FilterSlots::CHILD] + .as_ref() + .vortex_expect("no child for Filter"); + let Some(new_child) = scalar_fn_on_inner(inner, parent, child_idx, ctx)? else { return Ok(None); }; let mask = child.filter_mask().clone(); - let new_parent = FilterArray::try_new(new_child.clone(), mask)?; + let new_parent = FilterArray::try_new(new_child, mask)?; Ok(Some(new_parent.into_array())) } } @@ -254,13 +248,23 @@ impl ExecuteParentKernel for FilterSliceScalarFnUnaryPushDownRule { child_idx: usize, ctx: &mut ExecutionCtx, ) -> VortexResult> { - const SLOT: usize = SliceSlots::CHILD; - let Some(new_child) = - filter_slice_execute_parent::(child, parent, child_idx, ctx)? + let inner = child.slots()[SliceSlots::CHILD] + .as_ref() + .vortex_expect("no child for Slice"); + // Executing Slice on compressed form is beneficial because Slice + // doesn't canonicalize but shortens the range of data. + let kernels = Arc::clone(&ctx.execute_parent_kernels); + let Some(sliced) = execute_parent_for_child( + "filter_scalar_fn_pushdown", + child.array(), + inner, + SliceSlots::CHILD, + &kernels, + ctx, + )? else { return Ok(None); }; - let new_parent = SliceArray::try_new(new_child.clone(), child.slice_range().clone())?; - Ok(Some(new_parent.into_array())) + scalar_fn_on_inner(&sliced, parent, child_idx, ctx) } } From de0c70a865eda2af594e2b2b1999ee21c3502719 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Fri, 4 Sep 2026 18:20:18 +0100 Subject: [PATCH 5/6] resolve slice eagerly Signed-off-by: Mikhail Kot --- vortex-array/src/arrays/filter/kernel.rs | 43 ++---------------------- vortex-array/src/arrays/slice/mod.rs | 27 +++++++++++++++ vortex-layout/src/layouts/flat/reader.rs | 10 +++--- 3 files changed, 36 insertions(+), 44 deletions(-) diff --git a/vortex-array/src/arrays/filter/kernel.rs b/vortex-array/src/arrays/filter/kernel.rs index d4fc3f645bc..02c6ef43634 100644 --- a/vortex-array/src/arrays/filter/kernel.rs +++ b/vortex-array/src/arrays/filter/kernel.rs @@ -29,11 +29,9 @@ use crate::arrays::Filter; use crate::arrays::FilterArray; use crate::arrays::ScalarFn; use crate::arrays::ScalarFnArray; -use crate::arrays::Slice; use crate::arrays::dict::TakeExecuteAdaptor; use crate::arrays::filter::FilterSlots; use crate::arrays::scalar_fn::ScalarFnArrayExt; -use crate::arrays::slice::SliceSlots; use crate::execute_parent_for_child; use crate::kernel::ExecuteParentKernel; use crate::matcher::Matcher; @@ -49,12 +47,7 @@ pub(crate) fn initialize(session: &VortexSession) { kernels.register_execute_parent_kernel(Dict.id(), Filter, TakeExecuteAdaptor(Filter)); for parent in [Binary.id(), Between.id(), FillNull.id()] { - kernels.register_execute_parent_kernel( - parent, - Filter, - FilterSliceScalarFnUnaryPushDownRule, - ); - kernels.register_execute_parent_kernel(parent, Slice, FilterSliceScalarFnUnaryPushDownRule); + kernels.register_execute_parent_kernel(parent, Filter, FilterScalarFnUnaryPushDownRule); } } @@ -216,9 +209,9 @@ fn scalar_fn_on_inner( } #[derive(Debug)] -struct FilterSliceScalarFnUnaryPushDownRule; +struct FilterScalarFnUnaryPushDownRule; -impl ExecuteParentKernel for FilterSliceScalarFnUnaryPushDownRule { +impl ExecuteParentKernel for FilterScalarFnUnaryPushDownRule { type Parent = ScalarFn; fn execute_parent( &self, @@ -238,33 +231,3 @@ impl ExecuteParentKernel for FilterSliceScalarFnUnaryPushDownRule { Ok(Some(new_parent.into_array())) } } - -impl ExecuteParentKernel for FilterSliceScalarFnUnaryPushDownRule { - type Parent = ScalarFn; - fn execute_parent( - &self, - child: ArrayView<'_, Slice>, - parent: ArrayView<'_, ScalarFn>, - child_idx: usize, - ctx: &mut ExecutionCtx, - ) -> VortexResult> { - let inner = child.slots()[SliceSlots::CHILD] - .as_ref() - .vortex_expect("no child for Slice"); - // Executing Slice on compressed form is beneficial because Slice - // doesn't canonicalize but shortens the range of data. - let kernels = Arc::clone(&ctx.execute_parent_kernels); - let Some(sliced) = execute_parent_for_child( - "filter_scalar_fn_pushdown", - child.array(), - inner, - SliceSlots::CHILD, - &kernels, - ctx, - )? - else { - return Ok(None); - }; - scalar_fn_on_inner(&sliced, parent, child_idx, ctx) - } -} diff --git a/vortex-array/src/arrays/slice/mod.rs b/vortex-array/src/arrays/slice/mod.rs index bdbd75f45b6..699cfe1fde9 100644 --- a/vortex-array/src/arrays/slice/mod.rs +++ b/vortex-array/src/arrays/slice/mod.rs @@ -14,6 +14,7 @@ mod slice_; mod vtable; use std::ops::Range; +use std::sync::Arc; pub use array::SliceArraySlotsExt; pub use array::SliceData; @@ -29,6 +30,7 @@ use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::array::VTable; +use crate::execute_parent_for_child; use crate::kernel::ExecuteParentKernel; use crate::matcher::Matcher; use crate::optimizer::rules::ArrayParentReduceRule; @@ -104,6 +106,31 @@ where } } +/// Slice an array and eagerly resolve slice through encoding's kernels +pub fn slice_execute( + array: &ArrayRef, + range: Range, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let sliced = array.slice(range)?; + let Some(view) = sliced.as_opt::() else { + return Ok(sliced); + }; + let Some(inner) = view.slots()[SliceSlots::CHILD].as_ref().cloned() else { + return Ok(sliced); + }; + let kernels = Arc::clone(&ctx.execute_parent_kernels); + Ok(execute_parent_for_child( + "eager_slice", + &sliced, + &inner, + SliceSlots::CHILD, + &kernels, + ctx, + )? + .unwrap_or(sliced)) +} + /// Adaptor that wraps a [`SliceKernel`] impl as an [`ExecuteParentKernel`]. #[derive(Default, Debug)] pub struct SliceExecuteAdaptor(pub V); diff --git a/vortex-layout/src/layouts/flat/reader.rs b/vortex-layout/src/layouts/flat/reader.rs index aa7609f1659..46b903f0300 100644 --- a/vortex-layout/src/layouts/flat/reader.rs +++ b/vortex-layout/src/layouts/flat/reader.rs @@ -11,6 +11,7 @@ use tracing::trace; use vortex_array::ArrayRef; use vortex_array::MaskFuture; use vortex_array::VortexSessionExecute; +use vortex_array::arrays::slice::slice_executed; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::expr::BoundExpression; @@ -142,10 +143,11 @@ impl LayoutReader for FlatReader { // to evaluating the expression. let mut array = array.clone().await?; let mask = mask.await?; + let mut ctx = session.create_execution_ctx(); // Slice the array based on the row mask. if row_range.start > 0 || row_range.end < array.len() { - array = array.slice(row_range.clone())?; + array = slice_executed(&array, row_range.clone(), &mut ctx)?; } let mask_density = mask.density(); @@ -155,14 +157,12 @@ impl LayoutReader for FlatReader { // after this. let array = array.apply_bound(&expr)?; let array = array.filter(mask.clone())?; - let mut ctx = session.create_execution_ctx(); let array_mask = array.null_as_false().execute(&mut ctx)?; mask.intersect_by_rank(&array_mask) } else { // Run over the full array, with a simpler bitand at the end. let array = array.apply_bound(&expr)?; - let mut ctx = session.create_execution_ctx(); let array_mask = array.null_as_false().execute(&mut ctx)?; mask.bitand(&array_mask) @@ -193,6 +193,7 @@ impl LayoutReader for FlatReader { let name = Arc::clone(&self.name); let array = self.array_future(); let expr = expr.clone(); + let session = self.session.clone(); Ok(async move { trace!("Flat array evaluation {} - {}", name, expr); @@ -202,7 +203,8 @@ impl LayoutReader for FlatReader { // Slice the array based on the row mask. if row_range.start > 0 || row_range.end < array.len() { - array = array.slice(row_range.clone())?; + let mut ctx = session.create_execution_ctx(); + array = slice_executed(&array, row_range.clone(), &mut ctx)?; } // First apply the filter to the array. From 50bc21e7df0df6eb2b133efa8171d578391a70cc Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Fri, 4 Sep 2026 18:23:11 +0100 Subject: [PATCH 6/6] fix Signed-off-by: Mikhail Kot --- vortex-layout/src/layouts/flat/reader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vortex-layout/src/layouts/flat/reader.rs b/vortex-layout/src/layouts/flat/reader.rs index 46b903f0300..6d83afd6c2f 100644 --- a/vortex-layout/src/layouts/flat/reader.rs +++ b/vortex-layout/src/layouts/flat/reader.rs @@ -11,7 +11,7 @@ use tracing::trace; use vortex_array::ArrayRef; use vortex_array::MaskFuture; use vortex_array::VortexSessionExecute; -use vortex_array::arrays::slice::slice_executed; +use vortex_array::arrays::slice::slice_execute; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::expr::BoundExpression;