From f8c7de6a7574ca2662e78d5422b24a34a16eb709 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 21:42:34 +0000 Subject: [PATCH 1/5] Add Tiled array that repeats a child array n times TiledArray repeats its child end to end `count` times. Execution decodes the child once and appends it per tile via Chunked; scalar_at wraps the index; validity tiles the child's validity. Reduce rules collapse empty, single-count, constant-child and nested tiles, and slices within one tile or over whole tiles stay cheap. It is in-memory only and not registered for serialization. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018SWbEv9WiVtYt84zf3rXEu Signed-off-by: Claude --- vortex-array/src/arrays/mod.rs | 7 +- vortex-array/src/arrays/tiled/array.rs | 65 +++++++ vortex-array/src/arrays/tiled/mod.rs | 29 +++ vortex-array/src/arrays/tiled/rules.rs | 42 ++++ vortex-array/src/arrays/tiled/tests.rs | 233 ++++++++++++++++++++++ vortex-array/src/arrays/tiled/vtable.rs | 248 ++++++++++++++++++++++++ 6 files changed, 623 insertions(+), 1 deletion(-) create mode 100644 vortex-array/src/arrays/tiled/array.rs create mode 100644 vortex-array/src/arrays/tiled/mod.rs create mode 100644 vortex-array/src/arrays/tiled/rules.rs create mode 100644 vortex-array/src/arrays/tiled/tests.rs create mode 100644 vortex-array/src/arrays/tiled/vtable.rs diff --git a/vortex-array/src/arrays/mod.rs b/vortex-array/src/arrays/mod.rs index 0b5bdcf29df..2eabd8db43c 100644 --- a/vortex-array/src/arrays/mod.rs +++ b/vortex-array/src/arrays/mod.rs @@ -11,7 +11,8 @@ //! Utility and lazy arrays represent common transformations without immediately materializing //! their result. Examples include [`ChunkedArray`] for concatenation, [`ConstantArray`] for repeated //! values, [`DictArray`] for dictionary encoding, [`FilterArray`] for masked rows, [`SliceArray`] -//! for views, and [`ScalarFnArray`] for deferred scalar-function execution. +//! for views, [`TiledArray`] for an array repeated end to end, and [`ScalarFnArray`] for deferred +//! scalar-function execution. //! //! Some public arrays are primarily internal building blocks. Their constructors and extension //! traits document the stable contract; avoid depending on undocumented slot order or metadata @@ -118,6 +119,10 @@ pub mod struct_; pub use struct_::Struct; pub use struct_::StructArray; +pub mod tiled; +pub use tiled::Tiled; +pub use tiled::TiledArray; + pub mod union; pub use union::Union; pub use union::UnionArray; diff --git a/vortex-array/src/arrays/tiled/array.rs b/vortex-array/src/arrays/tiled/array.rs new file mode 100644 index 00000000000..376b5ed079a --- /dev/null +++ b/vortex-array/src/arrays/tiled/array.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Display; +use std::fmt::Formatter; + +use vortex_error::VortexResult; +use vortex_error::vortex_err; + +use crate::ArrayRef; +use crate::array::ArrayParts; +use crate::array_slots; +use crate::arrays::Tiled; +use crate::arrays::TiledArray; + +/// The child slots of a [`TiledArray`]. +#[array_slots(Tiled)] +pub struct TiledSlots { + /// The array that is repeated. + #[slot(0)] + pub child: ArrayRef, +} + +/// The data of a [`TiledArray`]: how many times its child is repeated. +#[derive(Clone, Debug)] +pub struct TiledData { + pub(super) count: usize, +} + +impl Display for TiledData { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "count: {}", self.count) + } +} + +impl TiledData { + /// Returns the number of times the child is repeated. + pub fn count(&self) -> usize { + self.count + } +} + +/// Returns the length of `count` copies of an array of `child_len` rows. +pub(super) fn tiled_len(child_len: usize, count: usize) -> VortexResult { + child_len.checked_mul(count).ok_or_else(|| { + vortex_err!("Tiled length overflows usize: {child_len} rows repeated {count} times") + }) +} + +impl TiledArray { + /// Creates an array that repeats `child` end to end `count` times, without reading any values. + /// + /// Returns an error if the length of the result overflows `usize`. + pub fn try_new(child: ArrayRef, count: usize) -> VortexResult { + let len = tiled_len(child.len(), count)?; + let dtype = child.dtype().clone(); + Self::try_from_parts(ArrayParts::new( + Tiled, + dtype, + len, + TiledData { count }, + TiledSlots { child }.into_slots(), + )) + } +} diff --git a/vortex-array/src/arrays/tiled/mod.rs b/vortex-array/src/arrays/tiled/mod.rs new file mode 100644 index 00000000000..fdcb02cc0ed --- /dev/null +++ b/vortex-array/src/arrays/tiled/mod.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! An in-memory encoding that repeats a child array end to end. +//! +//! A [`TiledArray`] with child `c` and count `n` has `c.len() * n` rows, and row `i` is row +//! `i % c.len()` of `c`. For example, tiling `[1, 2, 3]` twice gives `[1, 2, 3, 1, 2, 3]`. +//! +//! Execution decodes the child once and appends it `n` times, so the child is never decoded per +//! tile. +//! +//! `Tiled` has no wire form, so the default [`ArraySession`] does not register it, and an array +//! that holds a `Tiled` cannot be serialized. +//! +//! [`ArraySession`]: crate::session::ArraySession + +mod array; +mod rules; +mod vtable; + +pub use array::TiledArraySlotsExt; +pub use array::TiledData; +pub use array::TiledSlots; +pub use array::TiledSlotsView; +pub use vtable::Tiled; +pub use vtable::TiledArray; + +#[cfg(test)] +mod tests; diff --git a/vortex-array/src/arrays/tiled/rules.rs b/vortex-array/src/arrays/tiled/rules.rs new file mode 100644 index 00000000000..a09dbc86526 --- /dev/null +++ b/vortex-array/src/arrays/tiled/rules.rs @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::ops::Range; + +use vortex_error::VortexResult; + +use crate::ArrayRef; +use crate::IntoArray; +use crate::array::ArrayView; +use crate::arrays::Tiled; +use crate::arrays::TiledArray; +use crate::arrays::slice::SliceReduce; +use crate::arrays::slice::SliceReduceAdaptor; +use crate::arrays::tiled::TiledArraySlotsExt; +use crate::optimizer::rules::ParentRuleSet; + +pub(super) const PARENT_RULES: ParentRuleSet = + ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(Tiled))]); + +impl SliceReduce for Tiled { + fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { + // The range is non-empty, so the child is too. + let child = array.child(); + let child_len = child.len(); + let first_tile = range.start / child_len; + let offset = first_tile * child_len; + + // A range inside one tile is a slice of the child. + if range.end - offset <= child_len { + return Ok(Some(child.slice(range.start - offset..range.end - offset)?)); + } + + // A range of whole tiles is a shorter tiling of the child. + if range.start % child_len == 0 && range.end % child_len == 0 { + let count = range.len() / child_len; + return Ok(Some(TiledArray::try_new(child.clone(), count)?.into_array())); + } + + Ok(None) + } +} diff --git a/vortex-array/src/arrays/tiled/tests.rs b/vortex-array/src/arrays/tiled/tests.rs new file mode 100644 index 00000000000..7556aaaa8b5 --- /dev/null +++ b/vortex-array/src/arrays/tiled/tests.rs @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::ops::Range; +use std::sync::LazyLock; + +use rstest::rstest; +use vortex_buffer::buffer; +use vortex_error::VortexResult; +use vortex_session::VortexSession; + +use crate::ArrayRef; +use crate::IntoArray; +use crate::VortexSessionExecute; +use crate::arrays::Constant; +use crate::arrays::ConstantArray; +use crate::arrays::Primitive; +use crate::arrays::PrimitiveArray; +use crate::arrays::StructArray; +use crate::arrays::Tiled; +use crate::arrays::TiledArray; +use crate::arrays::VarBinViewArray; +use crate::arrays::tiled::TiledArraySlotsExt; +use crate::assert_arrays_eq; +use crate::builders::ArrayBuilder; +use crate::builders::builder_with_capacity; +use crate::optimizer::ArrayOptimizer; +use crate::scalar::Scalar; + +static SESSION: LazyLock = LazyLock::new(VortexSession::empty); + +fn tiled(child: impl IntoArray, count: usize) -> VortexResult { + Ok(TiledArray::try_new(child.into_array(), count)?.into_array()) +} + +#[test] +fn tiles_primitive() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let array = tiled(buffer![1i32, 2, 3], 3)?; + + assert_eq!(array.len(), 9); + assert_arrays_eq!( + array, + PrimitiveArray::from_iter([1i32, 2, 3, 1, 2, 3, 1, 2, 3]), + &mut ctx + ); + Ok(()) +} + +#[test] +fn tiles_nulls() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let array = tiled(PrimitiveArray::from_option_iter([Some(1i64), None]), 2)?; + + assert_arrays_eq!( + array, + PrimitiveArray::from_option_iter([Some(1i64), None, Some(1), None]), + &mut ctx + ); + assert_eq!(array.invalid_count(&mut ctx)?, 2); + Ok(()) +} + +#[test] +fn tiles_strings() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let child = VarBinViewArray::from_iter_nullable_str([Some("a"), None, Some("a long string")]); + let array = tiled(child, 2)?; + + assert_arrays_eq!( + array, + VarBinViewArray::from_iter_nullable_str([ + Some("a"), + None, + Some("a long string"), + Some("a"), + None, + Some("a long string"), + ]), + &mut ctx + ); + Ok(()) +} + +#[test] +fn tiles_struct() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let child = StructArray::try_from_iter([ + ("a", buffer![1u8, 2].into_array()), + ( + "b", + VarBinViewArray::from_iter_str(["x", "y"]).into_array(), + ), + ])?; + let expected = StructArray::try_from_iter([ + ("a", buffer![1u8, 2, 1, 2, 1, 2].into_array()), + ( + "b", + VarBinViewArray::from_iter_str(["x", "y", "x", "y", "x", "y"]).into_array(), + ), + ])?; + + assert_arrays_eq!(tiled(child, 3)?, expected, &mut ctx); + Ok(()) +} + +#[rstest] +#[case::zero_count(buffer![1i32, 2, 3].into_array(), 0)] +#[case::empty_child(PrimitiveArray::from_iter(Vec::::new()).into_array(), 4)] +fn tiles_to_empty(#[case] child: ArrayRef, #[case] count: usize) -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let array = tiled(child, count)?; + + assert!(array.is_empty()); + assert_arrays_eq!( + array, + PrimitiveArray::from_iter(Vec::::new()), + &mut ctx + ); + Ok(()) +} + +#[test] +fn tiles_single_row() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let array = tiled(PrimitiveArray::from_option_iter([Some(7u16)]), 4)?; + + assert_arrays_eq!( + array, + PrimitiveArray::from_option_iter([Some(7u16); 4]), + &mut ctx + ); + Ok(()) +} + +#[test] +fn scalar_at_wraps() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let array = tiled(buffer![10u32, 20, 30], 5)?; + + for (index, expected) in [(0, 10u32), (2, 30), (3, 10), (7, 20), (14, 30)] { + assert_eq!(array.execute_scalar(index, &mut ctx)?, Scalar::from(expected)); + } + Ok(()) +} + +#[test] +fn append_to_builder() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let array = tiled(buffer![1i8, 2], 3)?; + let mut builder = builder_with_capacity(array.dtype(), array.len() + 1); + builder.append_scalar(&Scalar::from(0i8))?; + array.append_to_builder(builder.as_mut(), &mut ctx)?; + + assert_arrays_eq!( + builder.finish(), + PrimitiveArray::from_iter([0i8, 1, 2, 1, 2, 1, 2]), + &mut ctx + ); + Ok(()) +} + +#[rstest] +#[case::within_first_tile(0..2, vec![0, 1])] +#[case::within_later_tile(5..8, vec![1, 2, 3])] +#[case::exactly_one_tile(4..8, vec![0, 1, 2, 3])] +#[case::whole_tiles(4..12, vec![0, 1, 2, 3, 0, 1, 2, 3])] +#[case::across_tiles(3..6, vec![3, 0, 1])] +#[case::across_many_tiles(1..11, vec![1, 2, 3, 0, 1, 2, 3, 0, 1, 2])] +fn slice(#[case] range: Range, #[case] expected: Vec) -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let array = tiled(buffer![0i32, 1, 2, 3], 3)?; + + assert_arrays_eq!( + array.slice(range)?, + PrimitiveArray::from_iter(expected), + &mut ctx + ); + Ok(()) +} + +#[test] +fn slice_of_whole_tiles_stays_tiled() -> VortexResult<()> { + let array = tiled(buffer![0i32, 1, 2, 3], 3)?; + let sliced = array.slice(4..12)?; + + let tiled = sliced.as_::(); + assert_eq!(tiled.count(), 2); + assert_eq!(tiled.child().len(), 4); + Ok(()) +} + +#[test] +fn reduce_single_count_to_child() -> VortexResult<()> { + let child = buffer![1i32, 2].into_array(); + let reduced = tiled(child.clone(), 1)?.optimize()?; + + assert!(reduced.is::()); + assert_eq!(reduced.len(), child.len()); + Ok(()) +} + +#[test] +fn reduce_constant_child() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let reduced = tiled(ConstantArray::new(5i32, 3), 4)?.optimize()?; + + assert!(reduced.is::()); + assert_arrays_eq!(reduced, PrimitiveArray::from_iter([5i32; 12]), &mut ctx); + Ok(()) +} + +#[test] +fn reduce_nested_tiles() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let reduced = tiled(tiled(buffer![1i32, 2], 2)?, 3)?.optimize()?; + + let flattened = reduced.as_::(); + assert_eq!(flattened.count(), 6); + assert!(!flattened.child().is::()); + assert_arrays_eq!( + reduced, + PrimitiveArray::from_iter([1i32, 2].repeat(6)), + &mut ctx + ); + Ok(()) +} + +#[test] +fn rejects_overflowing_length() { + let result = TiledArray::try_new(buffer![1u8, 2].into_array(), usize::MAX); + assert!(result.is_err()); +} diff --git a/vortex-array/src/arrays/tiled/vtable.rs b/vortex-array/src/arrays/tiled/vtable.rs new file mode 100644 index 00000000000..e0054c2cf40 --- /dev/null +++ b/vortex-array/src/arrays/tiled/vtable.rs @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::hash::Hash; +use std::hash::Hasher; +use std::iter; + +use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_error::vortex_ensure_eq; +use vortex_error::vortex_err; +use vortex_error::vortex_panic; +use vortex_session::VortexSession; +use vortex_session::registry::CachedId; + +use crate::AnyCanonical; +use crate::ArrayEq; +use crate::ArrayHash; +use crate::ArrayRef; +use crate::Canonical; +use crate::EqMode; +use crate::ExecutionCtx; +use crate::ExecutionResult; +use crate::IntoArray; +use crate::array::Array; +use crate::array::ArrayId; +use crate::array::ArrayParts; +use crate::array::ArrayView; +use crate::array::OperationsVTable; +use crate::array::ProbeState; +use crate::array::VTable; +use crate::array::ValidityVTable; +use crate::array::with_empty_buffers; +use crate::arrays::ChunkedArray; +use crate::arrays::Constant; +use crate::arrays::ConstantArray; +use crate::arrays::tiled::TiledArraySlotsExt; +use crate::arrays::tiled::TiledData; +use crate::arrays::tiled::TiledSlots; +use crate::arrays::tiled::array::tiled_len; +use crate::arrays::tiled::rules::PARENT_RULES; +use crate::buffer::BufferHandle; +use crate::builders::ArrayBuilder; +use crate::dtype::DType; +use crate::require_child; +use crate::scalar::Scalar; +use crate::serde::ArrayChildren; +use crate::validity::Validity; + +/// The encoding of a [`TiledArray`]. +#[derive(Clone, Debug)] +pub struct Tiled; + +/// A [`Tiled`]-encoded array, which repeats its child end to end. +pub type TiledArray = Array; + +impl ArrayHash for TiledData { + fn array_hash(&self, state: &mut H, _accuracy: EqMode) { + self.count.hash(state); + } +} + +impl ArrayEq for TiledData { + fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { + self.count == other.count + } +} + +impl VTable for Tiled { + type TypedArrayData = TiledData; + + type OperationsVTable = Self; + type ValidityVTable = Self; + + fn id(&self) -> ArrayId { + static ID: CachedId = CachedId::new("vortex.tiled"); + *ID + } + + fn validate( + &self, + data: &TiledData, + dtype: &DType, + len: usize, + slots: &[Option], + ) -> VortexResult<()> { + vortex_ensure_eq!(slots.len(), TiledSlots::COUNT); + let child = slots[TiledSlots::CHILD] + .as_ref() + .ok_or_else(|| vortex_err!("Tiled requires a child"))?; + vortex_ensure_eq!( + child.dtype(), + dtype, + "Tiled child dtype does not match outer dtype" + ); + vortex_ensure_eq!( + tiled_len(child.len(), data.count)?, + len, + "Tiled length does not match child length times count" + ); + Ok(()) + } + + fn nbuffers(_array: ArrayView<'_, Self>) -> usize { + 0 + } + + fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { + vortex_panic!("Tiled buffer index {idx} out of bounds") + } + + fn buffer_name(_array: ArrayView<'_, Self>, _idx: usize) -> Option { + None + } + + fn with_buffers( + &self, + array: ArrayView<'_, Self>, + buffers: &[BufferHandle], + ) -> VortexResult> { + with_empty_buffers(self, array, buffers) + } + + fn serialize( + _array: ArrayView<'_, Self>, + _session: &VortexSession, + ) -> VortexResult>> { + Ok(None) + } + + fn deserialize( + &self, + _dtype: &DType, + _len: usize, + _metadata: &[u8], + _buffers: &[BufferHandle], + _children: &dyn ArrayChildren, + _session: &VortexSession, + ) -> VortexResult> { + vortex_bail!("Tiled array is not serializable") + } + + fn append_to_builder( + array: ArrayView<'_, Self>, + builder: &mut dyn ArrayBuilder, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + if array.is_empty() { + return Ok(()); + } + let child = array + .child() + .clone() + .execute::(ctx)? + .into_array(); + for _ in 0..array.count { + child.append_to_builder(builder, ctx)?; + } + Ok(()) + } + + fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { + TiledSlots::NAMES[idx].to_string() + } + + fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { + if array.child().len() == 1 { + let scalar = array.child().execute_scalar(0, ctx)?; + return Ok(ExecutionResult::done( + ConstantArray::new(scalar, array.len()).into_array(), + )); + } + + let array = require_child!(array, array.child(), TiledSlots::CHILD => AnyCanonical); + + // Decode the child once, then let `Chunked` append the decoded child once per tile. + let tiles = iter::repeat_n(array.child().clone(), array.count); + let chunked = ChunkedArray::try_new(tiles, array.dtype().clone())?; + Ok(ExecutionResult::done(chunked.into_array())) + } + + fn reduce(array: ArrayView<'_, Self>) -> VortexResult> { + if array.is_empty() { + return Ok(Some(Canonical::empty(array.dtype()).into_array())); + } + if array.count == 1 { + return Ok(Some(array.child().clone())); + } + if let Some(constant) = array.child().as_opt::() { + return Ok(Some( + ConstantArray::new(constant.scalar().clone(), array.len()).into_array(), + )); + } + if let Some(inner) = array.child().as_opt::() { + let count = inner + .count + .checked_mul(array.count) + .ok_or_else(|| vortex_err!("Tiled count overflows usize"))?; + return Ok(Some( + TiledArray::try_new(inner.child().clone(), count)?.into_array(), + )); + } + Ok(None) + } + + fn reduce_parent( + array: ArrayView<'_, Self>, + parent: &ArrayRef, + child_idx: usize, + ) -> VortexResult> { + PARENT_RULES.evaluate(array, parent, child_idx) + } +} + +impl OperationsVTable for Tiled { + type ProbeState = (); + + fn probe_scalar( + state: &mut ProbeState<'_, Tiled>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let child = state + .slot(TiledSlots::CHILD)? + .ok_or_else(|| vortex_err!("Tiled child slot is missing"))?; + let child_index = index % child.len(); + child.execute_scalar(child_index, ctx) + } + + fn scalar_at( + array: ArrayView<'_, Tiled>, + index: usize, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + Self::probe_scalar(&mut ProbeState::once(array), index, ctx) + } +} + +impl ValidityVTable for Tiled { + fn validity(array: ArrayView<'_, Tiled>) -> VortexResult { + Ok(match array.child().validity()? { + Validity::Array(validity) => { + Validity::Array(TiledArray::try_new(validity, array.count)?.into_array()) + } + validity => validity, + }) + } +} From 2111795a0a9557ed65fe1513f662189779381c77 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Oct 2026 21:44:17 +0000 Subject: [PATCH 2/5] Use TiledArray for repeated fixed-size list elements The fixed-size list builder's repeated-list append and constant fixed-size list canonicalization both tiled one array by building a ChunkedArray of n clones of it. Build a TiledArray instead, which stores the count rather than n chunk references and offsets. The builder optimizes the result so a constant tile collapses into one constant. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018SWbEv9WiVtYt84zf3rXEu Signed-off-by: Claude --- .../src/arrays/constant/vtable/canonical.rs | 30 +++++++------------ vortex-array/src/builders/child.rs | 4 +-- vortex-array/src/builders/fixed_size_list.rs | 19 +++++------- 3 files changed, 20 insertions(+), 33 deletions(-) diff --git a/vortex-array/src/arrays/constant/vtable/canonical.rs b/vortex-array/src/arrays/constant/vtable/canonical.rs index 686e59df76e..064244e91d3 100644 --- a/vortex-array/src/arrays/constant/vtable/canonical.rs +++ b/vortex-array/src/arrays/constant/vtable/canonical.rs @@ -20,7 +20,6 @@ use crate::ExecutionCtx; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::BoolArray; -use crate::arrays::ChunkedArray; use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::DecimalArray; @@ -31,6 +30,7 @@ use crate::arrays::MapArray; use crate::arrays::NullArray; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; +use crate::arrays::TiledArray; use crate::arrays::UnionArray; use crate::arrays::VarBinViewArray; use crate::arrays::VariantArray; @@ -461,7 +461,7 @@ fn constant_canonical_fixed_size_list_array( /// /// A fixed-size list holds its elements back to back, so the rows cannot share one range the way a /// list view's can. Uniform elements stay a [`ConstantArray`] covering the whole run; otherwise the -/// row materializes once and the run chunks that single copy. +/// row materializes once and the run tiles that single copy. fn tile_fixed_size_list_elements( values: &[Scalar], element_dtype: &DType, @@ -490,14 +490,9 @@ fn tile_fixed_size_list_elements( return tile; } - // SAFETY: every chunk is `tile` itself, so they share its dtype and none is empty. - unsafe { - ChunkedArray::new_unchecked( - std::iter::repeat_n(tile, len).collect::>(), - element_dtype.clone(), - ) - } - .into_array() + TiledArray::try_new(tile, len) + .vortex_expect("tiled elements length is the run's elements length") + .into_array() } } } @@ -521,7 +516,6 @@ mod tests { use crate::IntoArray; use crate::VortexSessionExecute; use crate::arrays::BoolArray; - use crate::arrays::Chunked; use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::DecimalArray; @@ -531,14 +525,15 @@ mod tests { use crate::arrays::NullArray; use crate::arrays::PrimitiveArray; use crate::arrays::StructArray; + use crate::arrays::Tiled; use crate::arrays::VarBinArray; use crate::arrays::VarBinViewArray; - use crate::arrays::chunked::ChunkedArrayExt; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::arrays::fixed_size_list::FixedSizeListArraySlotsExt; use crate::arrays::listview::ListViewArraySlotsExt; use crate::arrays::listview::ListViewRebuildMode; use crate::arrays::struct_::StructArrayExt; + use crate::arrays::tiled::TiledArraySlotsExt; use crate::assert_arrays_eq; use crate::dtype::DType; use crate::dtype::DecimalDType; @@ -1072,7 +1067,7 @@ mod tests { } } - /// Mixed elements should materialize once, with every row chunking that one copy. + /// Mixed elements should materialize once, with every row tiling that one copy. #[test] fn test_canonicalize_fixed_size_list_tiles_one_copy_of_mixed_elements() { let mut ctx = SESSION.create_execution_ctx(); @@ -1092,16 +1087,13 @@ mod tests { .unwrap(); assert_eq!(canonical.elements().len(), 2 * LEN); + let tiled = canonical.elements().as_::(); assert_eq!( - canonical.elements().as_::().nchunks(), + tiled.count(), LEN, "every row should reference the same tile rather than hold its own copy", ); - let chunks = canonical.elements().as_::(); - let tile_ptr = chunks.chunk(0).buffer_handles()[0].as_host().as_ptr(); - for chunk in chunks.iter_chunks() { - assert_eq!(chunk.buffer_handles()[0].as_host().as_ptr(), tile_ptr); - } + assert_eq!(tiled.child().len(), 2); for index in [0, 1, LEN - 1] { assert_arrays_eq!( canonical.fixed_size_list_elements_at(index).unwrap(), diff --git a/vortex-array/src/builders/child.rs b/vortex-array/src/builders/child.rs index 923b7f76114..415a9a49288 100644 --- a/vortex-array/src/builders/child.rs +++ b/vortex-array/src/builders/child.rs @@ -66,8 +66,8 @@ impl ChildBuilder { /// /// An appended [`ChunkedArray`] stays one chunk rather than giving up its own. Unpacking it /// would spill a chunk per row into this child for a caller expressing a repeated run as - /// chunks - a tiled fixed-size list, say - and the child's chunk list is what every later - /// append and the final [`finish`](Self::finish) walk. Nesting costs one level, not one per + /// chunks, and the child's chunk list is what every later append and the final + /// [`finish`](Self::finish) walk. Nesting costs one level, not one per /// appended array. /// /// Nothing is decoded here, so `_ctx` goes unused; it stays in the signature so that the diff --git a/vortex-array/src/builders/fixed_size_list.rs b/vortex-array/src/builders/fixed_size_list.rs index 0584e138b5b..fc22f24fa38 100644 --- a/vortex-array/src/builders/fixed_size_list.rs +++ b/vortex-array/src/builders/fixed_size_list.rs @@ -14,8 +14,8 @@ use vortex_error::vortex_panic; use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; -use crate::arrays::ChunkedArray; use crate::arrays::FixedSizeListArray; +use crate::arrays::TiledArray; use crate::arrays::fixed_size_list::FixedSizeListArraySlotsExt; use crate::builders::ArrayBuilder; use crate::builders::ChildBuilder; @@ -24,6 +24,7 @@ use crate::builders::ValidityBuilder; use crate::canonical::Canonical; use crate::dtype::DType; use crate::dtype::Nullability; +use crate::optimizer::ArrayOptimizer; use crate::scalar::ListScalar; use crate::scalar::Scalar; @@ -136,8 +137,8 @@ impl FixedSizeListBuilder { /// A fixed-size list array holds its elements back to back, so `n` identical lists are the /// array's elements tiled `n` times - there is no layout that lets the rows share one range of /// elements the way a list view's can. The tiling costs nothing to build even so: the elements - /// go in as a [`ChunkedArray`] of `n` clones of the same array, so the tile's values are stored - /// once however many rows reference them, and the child holds the whole run as one chunk. + /// go in as a [`TiledArray`] of `array` repeated `n` times, so the tile's values are stored once + /// however many rows reference them, and the child holds the whole run as one chunk. /// /// A caller with a run of appends to make should hand over the same `array` each time rather /// than rebuild it, which is the whole reason this takes an array instead of a scalar. @@ -154,15 +155,9 @@ impl FixedSizeListBuilder { return Ok(()); } - // SAFETY: every chunk is `array` itself, so they share its dtype and none is empty. - let tiled = unsafe { - ChunkedArray::new_unchecked( - std::iter::repeat_n(array.clone(), n).collect::>(), - self.element_dtype().clone(), - ) - }; - self.elements_builder - .append_array(&tiled.into_array(), ctx)?; + // Optimizing collapses a constant tile into one constant run. + let tiled = TiledArray::try_new(array.clone(), n)?.into_array().optimize()?; + self.elements_builder.append_array(&tiled, ctx)?; self.nulls.append_n_non_nulls(n); Ok(()) From 756b5e2edc8209b944ce66f442b14097088d5e31 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 09:25:06 +0000 Subject: [PATCH 3/5] Rename Tiled array to Repeated Rename the Tiled encoding to Repeated: the module becomes arrays::repeated, the types RepeatedArray, RepeatedData and RepeatedSlots, and the encoding ID vortex.repeated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018SWbEv9WiVtYt84zf3rXEu Signed-off-by: Claude --- .../src/arrays/constant/vtable/canonical.rs | 16 ++-- vortex-array/src/arrays/mod.rs | 12 +-- .../src/arrays/{tiled => repeated}/array.rs | 32 ++++---- vortex-array/src/arrays/repeated/mod.rs | 29 +++++++ .../src/arrays/{tiled => repeated}/rules.rs | 22 +++--- .../src/arrays/{tiled => repeated}/tests.rs | 76 +++++++++--------- .../src/arrays/{tiled => repeated}/vtable.rs | 78 +++++++++---------- vortex-array/src/arrays/tiled/mod.rs | 29 ------- vortex-array/src/builders/fixed_size_list.rs | 10 +-- 9 files changed, 152 insertions(+), 152 deletions(-) rename vortex-array/src/arrays/{tiled => repeated}/array.rs (63%) create mode 100644 vortex-array/src/arrays/repeated/mod.rs rename vortex-array/src/arrays/{tiled => repeated}/rules.rs (63%) rename vortex-array/src/arrays/{tiled => repeated}/tests.rs (69%) rename vortex-array/src/arrays/{tiled => repeated}/vtable.rs (71%) delete mode 100644 vortex-array/src/arrays/tiled/mod.rs diff --git a/vortex-array/src/arrays/constant/vtable/canonical.rs b/vortex-array/src/arrays/constant/vtable/canonical.rs index 064244e91d3..99eb800de78 100644 --- a/vortex-array/src/arrays/constant/vtable/canonical.rs +++ b/vortex-array/src/arrays/constant/vtable/canonical.rs @@ -29,8 +29,8 @@ use crate::arrays::ListViewArray; use crate::arrays::MapArray; use crate::arrays::NullArray; use crate::arrays::PrimitiveArray; +use crate::arrays::RepeatedArray; use crate::arrays::StructArray; -use crate::arrays::TiledArray; use crate::arrays::UnionArray; use crate::arrays::VarBinViewArray; use crate::arrays::VariantArray; @@ -490,8 +490,8 @@ fn tile_fixed_size_list_elements( return tile; } - TiledArray::try_new(tile, len) - .vortex_expect("tiled elements length is the run's elements length") + RepeatedArray::try_new(tile, len) + .vortex_expect("repeated elements length is the run's elements length") .into_array() } } @@ -524,16 +524,16 @@ mod tests { use crate::arrays::ListViewArray; use crate::arrays::NullArray; use crate::arrays::PrimitiveArray; + use crate::arrays::Repeated; use crate::arrays::StructArray; - use crate::arrays::Tiled; use crate::arrays::VarBinArray; use crate::arrays::VarBinViewArray; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::arrays::fixed_size_list::FixedSizeListArraySlotsExt; use crate::arrays::listview::ListViewArraySlotsExt; use crate::arrays::listview::ListViewRebuildMode; + use crate::arrays::repeated::RepeatedArraySlotsExt; use crate::arrays::struct_::StructArrayExt; - use crate::arrays::tiled::TiledArraySlotsExt; use crate::assert_arrays_eq; use crate::dtype::DType; use crate::dtype::DecimalDType; @@ -1087,13 +1087,13 @@ mod tests { .unwrap(); assert_eq!(canonical.elements().len(), 2 * LEN); - let tiled = canonical.elements().as_::(); + let repeated = canonical.elements().as_::(); assert_eq!( - tiled.count(), + repeated.count(), LEN, "every row should reference the same tile rather than hold its own copy", ); - assert_eq!(tiled.child().len(), 2); + assert_eq!(repeated.child().len(), 2); for index in [0, 1, LEN - 1] { assert_arrays_eq!( canonical.fixed_size_list_elements_at(index).unwrap(), diff --git a/vortex-array/src/arrays/mod.rs b/vortex-array/src/arrays/mod.rs index 2eabd8db43c..5f1bf6117fe 100644 --- a/vortex-array/src/arrays/mod.rs +++ b/vortex-array/src/arrays/mod.rs @@ -11,8 +11,8 @@ //! Utility and lazy arrays represent common transformations without immediately materializing //! their result. Examples include [`ChunkedArray`] for concatenation, [`ConstantArray`] for repeated //! values, [`DictArray`] for dictionary encoding, [`FilterArray`] for masked rows, [`SliceArray`] -//! for views, [`TiledArray`] for an array repeated end to end, and [`ScalarFnArray`] for deferred -//! scalar-function execution. +//! for views, [`RepeatedArray`] for an array repeated end to end, and [`ScalarFnArray`] for +//! deferred scalar-function execution. //! //! Some public arrays are primarily internal building blocks. Their constructors and extension //! traits document the stable contract; avoid depending on undocumented slot order or metadata @@ -103,6 +103,10 @@ pub mod primitive; pub use primitive::Primitive; pub use primitive::PrimitiveArray; +pub mod repeated; +pub use repeated::Repeated; +pub use repeated::RepeatedArray; + pub mod runend; pub use runend::RunEnd; pub use runend::RunEndArray; @@ -119,10 +123,6 @@ pub mod struct_; pub use struct_::Struct; pub use struct_::StructArray; -pub mod tiled; -pub use tiled::Tiled; -pub use tiled::TiledArray; - pub mod union; pub use union::Union; pub use union::UnionArray; diff --git a/vortex-array/src/arrays/tiled/array.rs b/vortex-array/src/arrays/repeated/array.rs similarity index 63% rename from vortex-array/src/arrays/tiled/array.rs rename to vortex-array/src/arrays/repeated/array.rs index 376b5ed079a..cca937a59f3 100644 --- a/vortex-array/src/arrays/tiled/array.rs +++ b/vortex-array/src/arrays/repeated/array.rs @@ -10,30 +10,30 @@ use vortex_error::vortex_err; use crate::ArrayRef; use crate::array::ArrayParts; use crate::array_slots; -use crate::arrays::Tiled; -use crate::arrays::TiledArray; +use crate::arrays::Repeated; +use crate::arrays::RepeatedArray; -/// The child slots of a [`TiledArray`]. -#[array_slots(Tiled)] -pub struct TiledSlots { +/// The child slots of a [`RepeatedArray`]. +#[array_slots(Repeated)] +pub struct RepeatedSlots { /// The array that is repeated. #[slot(0)] pub child: ArrayRef, } -/// The data of a [`TiledArray`]: how many times its child is repeated. +/// The data of a [`RepeatedArray`]: how many times its child is repeated. #[derive(Clone, Debug)] -pub struct TiledData { +pub struct RepeatedData { pub(super) count: usize, } -impl Display for TiledData { +impl Display for RepeatedData { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "count: {}", self.count) } } -impl TiledData { +impl RepeatedData { /// Returns the number of times the child is repeated. pub fn count(&self) -> usize { self.count @@ -41,25 +41,25 @@ impl TiledData { } /// Returns the length of `count` copies of an array of `child_len` rows. -pub(super) fn tiled_len(child_len: usize, count: usize) -> VortexResult { +pub(super) fn repeated_len(child_len: usize, count: usize) -> VortexResult { child_len.checked_mul(count).ok_or_else(|| { - vortex_err!("Tiled length overflows usize: {child_len} rows repeated {count} times") + vortex_err!("Repeated length overflows usize: {child_len} rows repeated {count} times") }) } -impl TiledArray { +impl RepeatedArray { /// Creates an array that repeats `child` end to end `count` times, without reading any values. /// /// Returns an error if the length of the result overflows `usize`. pub fn try_new(child: ArrayRef, count: usize) -> VortexResult { - let len = tiled_len(child.len(), count)?; + let len = repeated_len(child.len(), count)?; let dtype = child.dtype().clone(); Self::try_from_parts(ArrayParts::new( - Tiled, + Repeated, dtype, len, - TiledData { count }, - TiledSlots { child }.into_slots(), + RepeatedData { count }, + RepeatedSlots { child }.into_slots(), )) } } diff --git a/vortex-array/src/arrays/repeated/mod.rs b/vortex-array/src/arrays/repeated/mod.rs new file mode 100644 index 00000000000..5ce3502d06d --- /dev/null +++ b/vortex-array/src/arrays/repeated/mod.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! An in-memory encoding that repeats a child array end to end. +//! +//! A [`RepeatedArray`] with child `c` and count `n` has `c.len() * n` rows, and row `i` is row +//! `i % c.len()` of `c`. For example, repeating `[1, 2, 3]` twice gives `[1, 2, 3, 1, 2, 3]`. +//! +//! Execution decodes the child once and appends it `n` times, so the child is never decoded per +//! copy. +//! +//! `Repeated` has no wire form, so the default [`ArraySession`] does not register it, and an array +//! that holds a `Repeated` cannot be serialized. +//! +//! [`ArraySession`]: crate::session::ArraySession + +mod array; +mod rules; +mod vtable; + +pub use array::RepeatedArraySlotsExt; +pub use array::RepeatedData; +pub use array::RepeatedSlots; +pub use array::RepeatedSlotsView; +pub use vtable::Repeated; +pub use vtable::RepeatedArray; + +#[cfg(test)] +mod tests; diff --git a/vortex-array/src/arrays/tiled/rules.rs b/vortex-array/src/arrays/repeated/rules.rs similarity index 63% rename from vortex-array/src/arrays/tiled/rules.rs rename to vortex-array/src/arrays/repeated/rules.rs index a09dbc86526..f68d1fe42ce 100644 --- a/vortex-array/src/arrays/tiled/rules.rs +++ b/vortex-array/src/arrays/repeated/rules.rs @@ -8,33 +8,33 @@ use vortex_error::VortexResult; use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; -use crate::arrays::Tiled; -use crate::arrays::TiledArray; +use crate::arrays::Repeated; +use crate::arrays::RepeatedArray; +use crate::arrays::repeated::RepeatedArraySlotsExt; use crate::arrays::slice::SliceReduce; use crate::arrays::slice::SliceReduceAdaptor; -use crate::arrays::tiled::TiledArraySlotsExt; use crate::optimizer::rules::ParentRuleSet; -pub(super) const PARENT_RULES: ParentRuleSet = - ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(Tiled))]); +pub(super) const PARENT_RULES: ParentRuleSet = + ParentRuleSet::new(&[ParentRuleSet::lift(&SliceReduceAdaptor(Repeated))]); -impl SliceReduce for Tiled { +impl SliceReduce for Repeated { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // The range is non-empty, so the child is too. let child = array.child(); let child_len = child.len(); - let first_tile = range.start / child_len; - let offset = first_tile * child_len; + let first_copy = range.start / child_len; + let offset = first_copy * child_len; - // A range inside one tile is a slice of the child. + // A range inside one copy is a slice of the child. if range.end - offset <= child_len { return Ok(Some(child.slice(range.start - offset..range.end - offset)?)); } - // A range of whole tiles is a shorter tiling of the child. + // A range of whole copies is a shorter repetition of the child. if range.start % child_len == 0 && range.end % child_len == 0 { let count = range.len() / child_len; - return Ok(Some(TiledArray::try_new(child.clone(), count)?.into_array())); + return Ok(Some(RepeatedArray::try_new(child.clone(), count)?.into_array())); } Ok(None) diff --git a/vortex-array/src/arrays/tiled/tests.rs b/vortex-array/src/arrays/repeated/tests.rs similarity index 69% rename from vortex-array/src/arrays/tiled/tests.rs rename to vortex-array/src/arrays/repeated/tests.rs index 7556aaaa8b5..ba6db486ef2 100644 --- a/vortex-array/src/arrays/tiled/tests.rs +++ b/vortex-array/src/arrays/repeated/tests.rs @@ -16,11 +16,11 @@ use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; +use crate::arrays::Repeated; +use crate::arrays::RepeatedArray; use crate::arrays::StructArray; -use crate::arrays::Tiled; -use crate::arrays::TiledArray; use crate::arrays::VarBinViewArray; -use crate::arrays::tiled::TiledArraySlotsExt; +use crate::arrays::repeated::RepeatedArraySlotsExt; use crate::assert_arrays_eq; use crate::builders::ArrayBuilder; use crate::builders::builder_with_capacity; @@ -29,14 +29,14 @@ use crate::scalar::Scalar; static SESSION: LazyLock = LazyLock::new(VortexSession::empty); -fn tiled(child: impl IntoArray, count: usize) -> VortexResult { - Ok(TiledArray::try_new(child.into_array(), count)?.into_array()) +fn repeated(child: impl IntoArray, count: usize) -> VortexResult { + Ok(RepeatedArray::try_new(child.into_array(), count)?.into_array()) } #[test] -fn tiles_primitive() -> VortexResult<()> { +fn repeats_primitive() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); - let array = tiled(buffer![1i32, 2, 3], 3)?; + let array = repeated(buffer![1i32, 2, 3], 3)?; assert_eq!(array.len(), 9); assert_arrays_eq!( @@ -48,9 +48,9 @@ fn tiles_primitive() -> VortexResult<()> { } #[test] -fn tiles_nulls() -> VortexResult<()> { +fn repeats_nulls() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); - let array = tiled(PrimitiveArray::from_option_iter([Some(1i64), None]), 2)?; + let array = repeated(PrimitiveArray::from_option_iter([Some(1i64), None]), 2)?; assert_arrays_eq!( array, @@ -62,10 +62,10 @@ fn tiles_nulls() -> VortexResult<()> { } #[test] -fn tiles_strings() -> VortexResult<()> { +fn repeats_strings() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); let child = VarBinViewArray::from_iter_nullable_str([Some("a"), None, Some("a long string")]); - let array = tiled(child, 2)?; + let array = repeated(child, 2)?; assert_arrays_eq!( array, @@ -83,7 +83,7 @@ fn tiles_strings() -> VortexResult<()> { } #[test] -fn tiles_struct() -> VortexResult<()> { +fn repeats_struct() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); let child = StructArray::try_from_iter([ ("a", buffer![1u8, 2].into_array()), @@ -100,16 +100,16 @@ fn tiles_struct() -> VortexResult<()> { ), ])?; - assert_arrays_eq!(tiled(child, 3)?, expected, &mut ctx); + assert_arrays_eq!(repeated(child, 3)?, expected, &mut ctx); Ok(()) } #[rstest] #[case::zero_count(buffer![1i32, 2, 3].into_array(), 0)] #[case::empty_child(PrimitiveArray::from_iter(Vec::::new()).into_array(), 4)] -fn tiles_to_empty(#[case] child: ArrayRef, #[case] count: usize) -> VortexResult<()> { +fn repeats_to_empty(#[case] child: ArrayRef, #[case] count: usize) -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); - let array = tiled(child, count)?; + let array = repeated(child, count)?; assert!(array.is_empty()); assert_arrays_eq!( @@ -121,9 +121,9 @@ fn tiles_to_empty(#[case] child: ArrayRef, #[case] count: usize) -> VortexResult } #[test] -fn tiles_single_row() -> VortexResult<()> { +fn repeats_single_row() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); - let array = tiled(PrimitiveArray::from_option_iter([Some(7u16)]), 4)?; + let array = repeated(PrimitiveArray::from_option_iter([Some(7u16)]), 4)?; assert_arrays_eq!( array, @@ -136,7 +136,7 @@ fn tiles_single_row() -> VortexResult<()> { #[test] fn scalar_at_wraps() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); - let array = tiled(buffer![10u32, 20, 30], 5)?; + let array = repeated(buffer![10u32, 20, 30], 5)?; for (index, expected) in [(0, 10u32), (2, 30), (3, 10), (7, 20), (14, 30)] { assert_eq!(array.execute_scalar(index, &mut ctx)?, Scalar::from(expected)); @@ -147,7 +147,7 @@ fn scalar_at_wraps() -> VortexResult<()> { #[test] fn append_to_builder() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); - let array = tiled(buffer![1i8, 2], 3)?; + let array = repeated(buffer![1i8, 2], 3)?; let mut builder = builder_with_capacity(array.dtype(), array.len() + 1); builder.append_scalar(&Scalar::from(0i8))?; array.append_to_builder(builder.as_mut(), &mut ctx)?; @@ -161,15 +161,15 @@ fn append_to_builder() -> VortexResult<()> { } #[rstest] -#[case::within_first_tile(0..2, vec![0, 1])] -#[case::within_later_tile(5..8, vec![1, 2, 3])] -#[case::exactly_one_tile(4..8, vec![0, 1, 2, 3])] -#[case::whole_tiles(4..12, vec![0, 1, 2, 3, 0, 1, 2, 3])] -#[case::across_tiles(3..6, vec![3, 0, 1])] -#[case::across_many_tiles(1..11, vec![1, 2, 3, 0, 1, 2, 3, 0, 1, 2])] +#[case::within_first_copy(0..2, vec![0, 1])] +#[case::within_later_copy(5..8, vec![1, 2, 3])] +#[case::exactly_one_copy(4..8, vec![0, 1, 2, 3])] +#[case::whole_copies(4..12, vec![0, 1, 2, 3, 0, 1, 2, 3])] +#[case::across_copies(3..6, vec![3, 0, 1])] +#[case::across_many_copies(1..11, vec![1, 2, 3, 0, 1, 2, 3, 0, 1, 2])] fn slice(#[case] range: Range, #[case] expected: Vec) -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); - let array = tiled(buffer![0i32, 1, 2, 3], 3)?; + let array = repeated(buffer![0i32, 1, 2, 3], 3)?; assert_arrays_eq!( array.slice(range)?, @@ -180,20 +180,20 @@ fn slice(#[case] range: Range, #[case] expected: Vec) -> VortexResul } #[test] -fn slice_of_whole_tiles_stays_tiled() -> VortexResult<()> { - let array = tiled(buffer![0i32, 1, 2, 3], 3)?; +fn slice_of_whole_copies_stays_repeated() -> VortexResult<()> { + let array = repeated(buffer![0i32, 1, 2, 3], 3)?; let sliced = array.slice(4..12)?; - let tiled = sliced.as_::(); - assert_eq!(tiled.count(), 2); - assert_eq!(tiled.child().len(), 4); + let repeated = sliced.as_::(); + assert_eq!(repeated.count(), 2); + assert_eq!(repeated.child().len(), 4); Ok(()) } #[test] fn reduce_single_count_to_child() -> VortexResult<()> { let child = buffer![1i32, 2].into_array(); - let reduced = tiled(child.clone(), 1)?.optimize()?; + let reduced = repeated(child.clone(), 1)?.optimize()?; assert!(reduced.is::()); assert_eq!(reduced.len(), child.len()); @@ -203,7 +203,7 @@ fn reduce_single_count_to_child() -> VortexResult<()> { #[test] fn reduce_constant_child() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); - let reduced = tiled(ConstantArray::new(5i32, 3), 4)?.optimize()?; + let reduced = repeated(ConstantArray::new(5i32, 3), 4)?.optimize()?; assert!(reduced.is::()); assert_arrays_eq!(reduced, PrimitiveArray::from_iter([5i32; 12]), &mut ctx); @@ -211,13 +211,13 @@ fn reduce_constant_child() -> VortexResult<()> { } #[test] -fn reduce_nested_tiles() -> VortexResult<()> { +fn reduce_nested_repeats() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); - let reduced = tiled(tiled(buffer![1i32, 2], 2)?, 3)?.optimize()?; + let reduced = repeated(repeated(buffer![1i32, 2], 2)?, 3)?.optimize()?; - let flattened = reduced.as_::(); + let flattened = reduced.as_::(); assert_eq!(flattened.count(), 6); - assert!(!flattened.child().is::()); + assert!(!flattened.child().is::()); assert_arrays_eq!( reduced, PrimitiveArray::from_iter([1i32, 2].repeat(6)), @@ -228,6 +228,6 @@ fn reduce_nested_tiles() -> VortexResult<()> { #[test] fn rejects_overflowing_length() { - let result = TiledArray::try_new(buffer![1u8, 2].into_array(), usize::MAX); + let result = RepeatedArray::try_new(buffer![1u8, 2].into_array(), usize::MAX); assert!(result.is_err()); } diff --git a/vortex-array/src/arrays/tiled/vtable.rs b/vortex-array/src/arrays/repeated/vtable.rs similarity index 71% rename from vortex-array/src/arrays/tiled/vtable.rs rename to vortex-array/src/arrays/repeated/vtable.rs index e0054c2cf40..997e8ae63aa 100644 --- a/vortex-array/src/arrays/tiled/vtable.rs +++ b/vortex-array/src/arrays/repeated/vtable.rs @@ -34,11 +34,11 @@ use crate::array::with_empty_buffers; use crate::arrays::ChunkedArray; use crate::arrays::Constant; use crate::arrays::ConstantArray; -use crate::arrays::tiled::TiledArraySlotsExt; -use crate::arrays::tiled::TiledData; -use crate::arrays::tiled::TiledSlots; -use crate::arrays::tiled::array::tiled_len; -use crate::arrays::tiled::rules::PARENT_RULES; +use crate::arrays::repeated::RepeatedArraySlotsExt; +use crate::arrays::repeated::RepeatedData; +use crate::arrays::repeated::RepeatedSlots; +use crate::arrays::repeated::array::repeated_len; +use crate::arrays::repeated::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::builders::ArrayBuilder; use crate::dtype::DType; @@ -47,56 +47,56 @@ use crate::scalar::Scalar; use crate::serde::ArrayChildren; use crate::validity::Validity; -/// The encoding of a [`TiledArray`]. +/// The encoding of a [`RepeatedArray`]. #[derive(Clone, Debug)] -pub struct Tiled; +pub struct Repeated; -/// A [`Tiled`]-encoded array, which repeats its child end to end. -pub type TiledArray = Array; +/// A [`Repeated`]-encoded array, which repeats its child end to end. +pub type RepeatedArray = Array; -impl ArrayHash for TiledData { +impl ArrayHash for RepeatedData { fn array_hash(&self, state: &mut H, _accuracy: EqMode) { self.count.hash(state); } } -impl ArrayEq for TiledData { +impl ArrayEq for RepeatedData { fn array_eq(&self, other: &Self, _accuracy: EqMode) -> bool { self.count == other.count } } -impl VTable for Tiled { - type TypedArrayData = TiledData; +impl VTable for Repeated { + type TypedArrayData = RepeatedData; type OperationsVTable = Self; type ValidityVTable = Self; fn id(&self) -> ArrayId { - static ID: CachedId = CachedId::new("vortex.tiled"); + static ID: CachedId = CachedId::new("vortex.repeated"); *ID } fn validate( &self, - data: &TiledData, + data: &RepeatedData, dtype: &DType, len: usize, slots: &[Option], ) -> VortexResult<()> { - vortex_ensure_eq!(slots.len(), TiledSlots::COUNT); - let child = slots[TiledSlots::CHILD] + vortex_ensure_eq!(slots.len(), RepeatedSlots::COUNT); + let child = slots[RepeatedSlots::CHILD] .as_ref() - .ok_or_else(|| vortex_err!("Tiled requires a child"))?; + .ok_or_else(|| vortex_err!("Repeated requires a child"))?; vortex_ensure_eq!( child.dtype(), dtype, - "Tiled child dtype does not match outer dtype" + "Repeated child dtype does not match outer dtype" ); vortex_ensure_eq!( - tiled_len(child.len(), data.count)?, + repeated_len(child.len(), data.count)?, len, - "Tiled length does not match child length times count" + "Repeated length does not match child length times count" ); Ok(()) } @@ -106,7 +106,7 @@ impl VTable for Tiled { } fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { - vortex_panic!("Tiled buffer index {idx} out of bounds") + vortex_panic!("Repeated buffer index {idx} out of bounds") } fn buffer_name(_array: ArrayView<'_, Self>, _idx: usize) -> Option { @@ -137,7 +137,7 @@ impl VTable for Tiled { _children: &dyn ArrayChildren, _session: &VortexSession, ) -> VortexResult> { - vortex_bail!("Tiled array is not serializable") + vortex_bail!("Repeated array is not serializable") } fn append_to_builder( @@ -160,7 +160,7 @@ impl VTable for Tiled { } fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { - TiledSlots::NAMES[idx].to_string() + RepeatedSlots::NAMES[idx].to_string() } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { @@ -171,11 +171,11 @@ impl VTable for Tiled { )); } - let array = require_child!(array, array.child(), TiledSlots::CHILD => AnyCanonical); + let array = require_child!(array, array.child(), RepeatedSlots::CHILD => AnyCanonical); - // Decode the child once, then let `Chunked` append the decoded child once per tile. - let tiles = iter::repeat_n(array.child().clone(), array.count); - let chunked = ChunkedArray::try_new(tiles, array.dtype().clone())?; + // Decode the child once, then let `Chunked` append the decoded child once per copy. + let copies = iter::repeat_n(array.child().clone(), array.count); + let chunked = ChunkedArray::try_new(copies, array.dtype().clone())?; Ok(ExecutionResult::done(chunked.into_array())) } @@ -191,13 +191,13 @@ impl VTable for Tiled { ConstantArray::new(constant.scalar().clone(), array.len()).into_array(), )); } - if let Some(inner) = array.child().as_opt::() { + if let Some(inner) = array.child().as_opt::() { let count = inner .count .checked_mul(array.count) - .ok_or_else(|| vortex_err!("Tiled count overflows usize"))?; + .ok_or_else(|| vortex_err!("Repeated count overflows usize"))?; return Ok(Some( - TiledArray::try_new(inner.child().clone(), count)?.into_array(), + RepeatedArray::try_new(inner.child().clone(), count)?.into_array(), )); } Ok(None) @@ -212,23 +212,23 @@ impl VTable for Tiled { } } -impl OperationsVTable for Tiled { +impl OperationsVTable for Repeated { type ProbeState = (); fn probe_scalar( - state: &mut ProbeState<'_, Tiled>, + state: &mut ProbeState<'_, Repeated>, index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { let child = state - .slot(TiledSlots::CHILD)? - .ok_or_else(|| vortex_err!("Tiled child slot is missing"))?; + .slot(RepeatedSlots::CHILD)? + .ok_or_else(|| vortex_err!("Repeated child slot is missing"))?; let child_index = index % child.len(); child.execute_scalar(child_index, ctx) } fn scalar_at( - array: ArrayView<'_, Tiled>, + array: ArrayView<'_, Repeated>, index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { @@ -236,11 +236,11 @@ impl OperationsVTable for Tiled { } } -impl ValidityVTable for Tiled { - fn validity(array: ArrayView<'_, Tiled>) -> VortexResult { +impl ValidityVTable for Repeated { + fn validity(array: ArrayView<'_, Repeated>) -> VortexResult { Ok(match array.child().validity()? { Validity::Array(validity) => { - Validity::Array(TiledArray::try_new(validity, array.count)?.into_array()) + Validity::Array(RepeatedArray::try_new(validity, array.count)?.into_array()) } validity => validity, }) diff --git a/vortex-array/src/arrays/tiled/mod.rs b/vortex-array/src/arrays/tiled/mod.rs deleted file mode 100644 index fdcb02cc0ed..00000000000 --- a/vortex-array/src/arrays/tiled/mod.rs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -//! An in-memory encoding that repeats a child array end to end. -//! -//! A [`TiledArray`] with child `c` and count `n` has `c.len() * n` rows, and row `i` is row -//! `i % c.len()` of `c`. For example, tiling `[1, 2, 3]` twice gives `[1, 2, 3, 1, 2, 3]`. -//! -//! Execution decodes the child once and appends it `n` times, so the child is never decoded per -//! tile. -//! -//! `Tiled` has no wire form, so the default [`ArraySession`] does not register it, and an array -//! that holds a `Tiled` cannot be serialized. -//! -//! [`ArraySession`]: crate::session::ArraySession - -mod array; -mod rules; -mod vtable; - -pub use array::TiledArraySlotsExt; -pub use array::TiledData; -pub use array::TiledSlots; -pub use array::TiledSlotsView; -pub use vtable::Tiled; -pub use vtable::TiledArray; - -#[cfg(test)] -mod tests; diff --git a/vortex-array/src/builders/fixed_size_list.rs b/vortex-array/src/builders/fixed_size_list.rs index fc22f24fa38..3f204463892 100644 --- a/vortex-array/src/builders/fixed_size_list.rs +++ b/vortex-array/src/builders/fixed_size_list.rs @@ -15,7 +15,7 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::FixedSizeListArray; -use crate::arrays::TiledArray; +use crate::arrays::RepeatedArray; use crate::arrays::fixed_size_list::FixedSizeListArraySlotsExt; use crate::builders::ArrayBuilder; use crate::builders::ChildBuilder; @@ -137,8 +137,8 @@ impl FixedSizeListBuilder { /// A fixed-size list array holds its elements back to back, so `n` identical lists are the /// array's elements tiled `n` times - there is no layout that lets the rows share one range of /// elements the way a list view's can. The tiling costs nothing to build even so: the elements - /// go in as a [`TiledArray`] of `array` repeated `n` times, so the tile's values are stored once - /// however many rows reference them, and the child holds the whole run as one chunk. + /// go in as a [`RepeatedArray`] of `array` repeated `n` times, so the tile's values are stored + /// once however many rows reference them, and the child holds the whole run as one chunk. /// /// A caller with a run of appends to make should hand over the same `array` each time rather /// than rebuild it, which is the whole reason this takes an array instead of a scalar. @@ -156,8 +156,8 @@ impl FixedSizeListBuilder { } // Optimizing collapses a constant tile into one constant run. - let tiled = TiledArray::try_new(array.clone(), n)?.into_array().optimize()?; - self.elements_builder.append_array(&tiled, ctx)?; + let repeated = RepeatedArray::try_new(array.clone(), n)?.into_array().optimize()?; + self.elements_builder.append_array(&repeated, ctx)?; self.nulls.append_n_non_nulls(n); Ok(()) From 3b02e6e71b91d489423be866daca9f9a4f520581 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 18:35:13 +0000 Subject: [PATCH 4/5] Fix Repeated array build, clippy and formatting Take the child length from the array rather than the probe, which has no len(); use is_multiple_of; build the test builder with an explicit allocator; and apply rustfmt. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018SWbEv9WiVtYt84zf3rXEu Signed-off-by: Claude --- vortex-array/src/arrays/repeated/rules.rs | 6 ++++-- vortex-array/src/arrays/repeated/tests.rs | 15 +++++++-------- vortex-array/src/arrays/repeated/vtable.rs | 8 ++++---- vortex-array/src/builders/fixed_size_list.rs | 4 +++- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/vortex-array/src/arrays/repeated/rules.rs b/vortex-array/src/arrays/repeated/rules.rs index f68d1fe42ce..e4562288974 100644 --- a/vortex-array/src/arrays/repeated/rules.rs +++ b/vortex-array/src/arrays/repeated/rules.rs @@ -32,9 +32,11 @@ impl SliceReduce for Repeated { } // A range of whole copies is a shorter repetition of the child. - if range.start % child_len == 0 && range.end % child_len == 0 { + if range.start.is_multiple_of(child_len) && range.end.is_multiple_of(child_len) { let count = range.len() / child_len; - return Ok(Some(RepeatedArray::try_new(child.clone(), count)?.into_array())); + return Ok(Some( + RepeatedArray::try_new(child.clone(), count)?.into_array(), + )); } Ok(None) diff --git a/vortex-array/src/arrays/repeated/tests.rs b/vortex-array/src/arrays/repeated/tests.rs index ba6db486ef2..722a6b1506a 100644 --- a/vortex-array/src/arrays/repeated/tests.rs +++ b/vortex-array/src/arrays/repeated/tests.rs @@ -22,8 +22,7 @@ use crate::arrays::StructArray; use crate::arrays::VarBinViewArray; use crate::arrays::repeated::RepeatedArraySlotsExt; use crate::assert_arrays_eq; -use crate::builders::ArrayBuilder; -use crate::builders::builder_with_capacity; +use crate::builders::builder_with_capacity_in; use crate::optimizer::ArrayOptimizer; use crate::scalar::Scalar; @@ -87,10 +86,7 @@ fn repeats_struct() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); let child = StructArray::try_from_iter([ ("a", buffer![1u8, 2].into_array()), - ( - "b", - VarBinViewArray::from_iter_str(["x", "y"]).into_array(), - ), + ("b", VarBinViewArray::from_iter_str(["x", "y"]).into_array()), ])?; let expected = StructArray::try_from_iter([ ("a", buffer![1u8, 2, 1, 2, 1, 2].into_array()), @@ -139,7 +135,10 @@ fn scalar_at_wraps() -> VortexResult<()> { let array = repeated(buffer![10u32, 20, 30], 5)?; for (index, expected) in [(0, 10u32), (2, 30), (3, 10), (7, 20), (14, 30)] { - assert_eq!(array.execute_scalar(index, &mut ctx)?, Scalar::from(expected)); + assert_eq!( + array.execute_scalar(index, &mut ctx)?, + Scalar::from(expected) + ); } Ok(()) } @@ -148,7 +147,7 @@ fn scalar_at_wraps() -> VortexResult<()> { fn append_to_builder() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); let array = repeated(buffer![1i8, 2], 3)?; - let mut builder = builder_with_capacity(array.dtype(), array.len() + 1); + let mut builder = builder_with_capacity_in(array.dtype(), array.len() + 1, ctx.allocator()); builder.append_scalar(&Scalar::from(0i8))?; array.append_to_builder(builder.as_mut(), &mut ctx)?; diff --git a/vortex-array/src/arrays/repeated/vtable.rs b/vortex-array/src/arrays/repeated/vtable.rs index 997e8ae63aa..a5cb2281212 100644 --- a/vortex-array/src/arrays/repeated/vtable.rs +++ b/vortex-array/src/arrays/repeated/vtable.rs @@ -220,11 +220,11 @@ impl OperationsVTable for Repeated { index: usize, ctx: &mut ExecutionCtx, ) -> VortexResult { - let child = state + let child_index = index % state.array().child().len(); + state .slot(RepeatedSlots::CHILD)? - .ok_or_else(|| vortex_err!("Repeated child slot is missing"))?; - let child_index = index % child.len(); - child.execute_scalar(child_index, ctx) + .ok_or_else(|| vortex_err!("Repeated child slot is missing"))? + .execute_scalar(child_index, ctx) } fn scalar_at( diff --git a/vortex-array/src/builders/fixed_size_list.rs b/vortex-array/src/builders/fixed_size_list.rs index 3f204463892..6a230f99ccd 100644 --- a/vortex-array/src/builders/fixed_size_list.rs +++ b/vortex-array/src/builders/fixed_size_list.rs @@ -156,7 +156,9 @@ impl FixedSizeListBuilder { } // Optimizing collapses a constant tile into one constant run. - let repeated = RepeatedArray::try_new(array.clone(), n)?.into_array().optimize()?; + let repeated = RepeatedArray::try_new(array.clone(), n)? + .into_array() + .optimize()?; self.elements_builder.append_array(&repeated, ctx)?; self.nulls.append_n_non_nulls(n); From 10fc9cebdd0a0f4ce8f546b41c8edc7f9c1dfe58 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 18:43:51 +0000 Subject: [PATCH 5/5] Expect constant elements for a uniform repeated fixed-size list Optimizing the repeated elements collapses a repeated constant tile into one constant array, so the test asserts the elements are constant rather than a chunked array of constants. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018SWbEv9WiVtYt84zf3rXEu Signed-off-by: Claude --- vortex-array/src/arrays/constant/vtable/mod.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index 3b55071ffb8..804b3591cf2 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -424,14 +424,12 @@ mod tests { use crate::ExecutionCtx; use crate::IntoArray; use crate::VortexSessionExecute; - use crate::arrays::Chunked; use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Extension; use crate::arrays::FixedSizeList; use crate::arrays::ListView; use crate::arrays::Struct; - use crate::arrays::chunked::ChunkedArrayExt; use crate::arrays::constant::vtable::canonical::constant_canonicalize; use crate::arrays::extension::ExtensionArrayExt; use crate::arrays::fixed_size_list::FixedSizeListArraySlotsExt; @@ -752,8 +750,8 @@ mod tests { assert_append_matches_canonical(ConstantArray::new(Scalar::null(dtype), 3)) } - /// A fixed-size list whose elements are all the same scalar tiles a constant array, so the - /// tile's chunks stay constant-encoded rather than materializing a value per row. + /// A fixed-size list whose elements are all the same scalar repeats a constant array, so the + /// elements collapse into one constant array rather than materializing a value per row. #[test] fn test_uniform_fixed_size_list_constant_append_keeps_elements_constant() -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); @@ -777,12 +775,10 @@ mod tests { let elements = result.as_::().elements().clone(); assert!( - elements - .as_::() - .iter_chunks() - .all(|chunk| chunk.is::()), + elements.is::(), "a uniform tile should have stayed constant-encoded", ); + assert_eq!(elements.len(), 2_000); Ok(()) }