From 10654ef369959ed14af27f5907602eedd1d9f05c Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sat, 15 Aug 2026 16:48:51 -0700 Subject: [PATCH 01/24] feat(transaction): add the Transaction V2 action vocabulary Introduces the Rust side of the action-based transaction draft: `Ref`, `UserOperation`, `UserAction`, and the eight `Action` variants this build implements (AddFragment, AddDataFile, AddField, AddBase, TombstoneFieldData, RemoveFragment, SetDeletionFile, AlterField). Types only -- no wire conversion, apply, or conflict handling yet, so nothing reaches these from the commit path. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance-table/src/transaction.rs | 2 + rust/lance-table/src/transaction/action.rs | 351 +++++++++++++++++++++ 2 files changed, 353 insertions(+) create mode 100644 rust/lance-table/src/transaction/action.rs diff --git a/rust/lance-table/src/transaction.rs b/rust/lance-table/src/transaction.rs index 74fba1b33ba..145886ca9f0 100644 --- a/rust/lance-table/src/transaction.rs +++ b/rust/lance-table/src/transaction.rs @@ -17,6 +17,7 @@ //! ```text //! builder Transaction: an operation plus the version it was based on //! operation the vocabulary of changes an operation can describe +//! action the finer-grained Transaction V2 vocabulary (draft) //! update_map incremental edits to the manifest's string maps //! validate pre-commit checks against the manifest being replaced //! manifest_build applying an operation to produce the next manifest @@ -26,6 +27,7 @@ //! proto the persisted protobuf encoding of all of the above //! ``` +pub mod action; mod builder; mod conflicts; mod index_maintenance; diff --git a/rust/lance-table/src/transaction/action.rs b/rust/lance-table/src/transaction/action.rs new file mode 100644 index 00000000000..28137675b69 --- /dev/null +++ b/rust/lance-table/src/transaction/action.rs @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! The action vocabulary of Transaction V2. +//! +//! Where an [`Operation`](super::Operation) names one whole change and carries a +//! post-image of the parts of the manifest it touches, a [`UserOperation`] is an +//! ordered list of [`Action`]s, each recording a single *delta*. Composing +//! several changes into one atomic commit and replaying a change onto a +//! different version both fall out of that, with no per-operation logic. +//! +//! The wire format and the reasoning behind it live in +//! `protos/transaction/actions.proto`; the two definitions must stay in step. +//! Only the subset of the drafted vocabulary that is implemented appears here -- +//! an action this build does not know is rejected on load rather than skipped. +//! +//! ```text +//! action the vocabulary (this module) +//! ``` +//! +//! # Stability +//! +//! Transaction V2 is a pre-vote draft. Nothing in this module is a compatibility +//! contract, and a transaction carrying a [`UserOperation`] is rejected outright +//! by libraries that predate it. + +use crate::format::{BasePath, DataFile, DeletionFile, RowIdMeta}; +use crate::rowids::version::RowDatasetVersionMeta; +use lance_core::datatypes::Field; +use lance_core::deepsize::DeepSizeOf; + +/// A reference to a counter-allocated identifier -- a field id, fragment id, or +/// base id -- that may not have been assigned yet. +/// +/// [`Ref::Committed`] is a concrete id that already exists in the manifest. +/// [`Ref::Local`] is a placeholder token minted by an `Add*` action earlier in +/// the same [`UserOperation`]; it resolves to a freshly-allocated id at apply, +/// and re-resolves against the target's counters when the operation is replayed +/// onto a newer version. That re-resolution is what lets two independent +/// `AddField`s on divergent branches become two distinct fields rather than a +/// collision. +/// +/// Local tokens are scoped to one [`UserOperation`] and must be distinct within +/// it. The three id spaces do not share a token namespace: a fragment token 0 +/// and a field token 0 are unrelated. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, DeepSizeOf)] +pub enum Ref { + Committed(u64), + Local(u32), +} + +impl Ref { + /// The committed id, or `None` if this is an unresolved local token. + pub fn committed(&self) -> Option { + match self { + Self::Committed(id) => Some(*id), + Self::Local(_) => None, + } + } + + /// The local token, or `None` if this reference is already committed. + pub fn local(&self) -> Option { + match self { + Self::Local(token) => Some(*token), + Self::Committed(_) => None, + } + } +} + +/// A composable transaction: an ordered list of user actions that commit +/// atomically as a single manifest change. +/// +/// The `uuid` and `read_version` carried on the wire mirror the enclosing +/// [`Transaction`](super::Transaction) and are filled in from it, so they are +/// not repeated here. +#[derive(Debug, Clone, PartialEq, DeepSizeOf, Default)] +pub struct UserOperation { + /// Human-readable description of the whole operation, e.g. `"INSERT INTO t"`. + pub description: String, + /// The ordered steps this operation applies. + pub actions: Vec, +} + +impl UserOperation { + pub fn new(description: impl Into, actions: Vec) -> Self { + Self { + description: description.into(), + actions, + } + } + + /// Every action in every step, in application order. + pub fn iter_actions(&self) -> impl Iterator { + self.actions.iter().flat_map(|step| step.actions.iter()) + } +} + +/// A single user-recognizable step within a [`UserOperation`], e.g. "append +/// batch" or "rebuild index". +/// +/// The description keeps transaction history readable: when a range of versions +/// is squashed, each original operation collapses into one step, so the sequence +/// a user performed survives even though the deltas are flattened when applied. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct UserAction { + pub description: String, + pub actions: Vec, +} + +impl UserAction { + pub fn new(description: impl Into, actions: Vec) -> Self { + Self { + description: description.into(), + actions, + } + } +} + +/// A single granular change to the manifest. +/// +/// The drafted vocabulary is larger than this; the variants here are the ones +/// this build implements end to end. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub enum Action { + AddFragment(AddFragment), + AddDataFile(AddDataFile), + AddField(AddField), + AddBase(AddBase), + TombstoneFieldData(TombstoneFieldData), + RemoveFragment(RemoveFragment), + SetDeletionFile(SetDeletionFile), + AlterField(AlterField), +} + +impl Action { + pub fn name(&self) -> &'static str { + match self { + Self::AddFragment(_) => "AddFragment", + Self::AddDataFile(_) => "AddDataFile", + Self::AddField(_) => "AddField", + Self::AddBase(_) => "AddBase", + Self::TombstoneFieldData(_) => "TombstoneFieldData", + Self::RemoveFragment(_) => "RemoveFragment", + Self::SetDeletionFile(_) => "SetDeletionFile", + Self::AlterField(_) => "AlterField", + } + } + + /// Whether this action changes the data a reader would see, as opposed to + /// rearranging how it is stored (compaction, a segment rebuild). + /// + /// CDC and streaming consumers use this to skip commits that cannot have + /// changed any row's value. + pub fn is_data_change(&self) -> bool { + match self { + Self::AddFragment(action) => action.data_change, + Self::AddDataFile(action) => action.data_change, + Self::TombstoneFieldData(action) => action.data_change, + Self::RemoveFragment(action) => action.data_change, + Self::SetDeletionFile(action) => action.data_change, + // Schema and base-path changes touch no row values. + Self::AddField(_) | Self::AddBase(_) | Self::AlterField(_) => false, + } + } +} + +impl std::fmt::Display for Action { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.name()) + } +} + +/// Mint a new, empty fragment. +/// +/// Its data files arrive via [`AddDataFile`] actions naming this fragment's +/// local token. A freshly-minted fragment has no deletion vector: it has no +/// committed rows to delete yet. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct AddFragment { + /// Token standing in for the fragment id until it is allocated at apply. + pub local: u32, + /// Physical rows in the fragment, including rows later tombstoned. + pub physical_rows: u64, + /// Stable row id sequence. `None` on datasets without stable row ids, and + /// on datasets that have them but where the ids are assigned at apply. + pub row_id_meta: Option, + /// Per-row version metadata, carried exactly as on + /// [`Fragment`](crate::format::Fragment). `None` means "stamp at apply". + pub last_updated_at_version_meta: Option, + pub created_at_version_meta: Option, + /// `false` marks a pure rearrangement, e.g. a compaction rewrite. + pub data_change: bool, +} + +/// Add a data file to a fragment. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct AddDataFile { + /// The fragment to add the file to: committed, or a fragment minted earlier + /// in the same operation. + pub fragment: Ref, + /// The file. Its `fields` are placeholders and are stamped in at apply from + /// `field_ids`, which is the authority for the column -> field mapping. + pub file: DataFile, + /// One entry per column in `file`, in column order. + pub field_ids: Vec, + /// See [`AddFragment::data_change`]. + pub data_change: bool, +} + +/// Mint a new schema field. +/// +/// A nested column that introduces several fields is several ordered +/// `AddField`s -- parent first, each child naming its parent's local token. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct AddField { + /// Token standing in for the field id until it is allocated at apply. + pub local: u32, + /// The parent field, or `None` for a top-level column. + pub parent: Option, + /// The field definition. Its `id`, `parent_id`, and `children` are ignored: + /// `local` and `parent` carry that structure, and each child is its own + /// action. + pub def: Field, +} + +/// Mint a new base path. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct AddBase { + /// Token standing in for the base id until it is allocated at apply. + pub local: u32, + /// The base path. Its `id` is ignored and stamped in at apply. + pub base: BasePath, +} + +/// Tombstone the data-file binding of committed fields within one fragment. +/// +/// Each field's slot in whatever file currently backs it is marked tombstoned, +/// and a file left with no live field is pruned at apply. Data files have no id +/// of their own and a live field is backed by exactly one file, so this is how a +/// column's data is dropped or superseded: re-encoding a column is a tombstone +/// followed by an [`AddDataFile`] for the same field. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct TombstoneFieldData { + pub fragment: Ref, + /// Committed field ids whose current backing is tombstoned. + pub field_ids: Vec, + /// See [`AddFragment::data_change`]. + pub data_change: bool, +} + +/// Remove a fragment entirely -- every row deleted, or the fragment replaced by +/// compaction. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct RemoveFragment { + pub fragment: Ref, + /// See [`AddFragment::data_change`]. + pub data_change: bool, +} + +/// Set (replace) a fragment's deletion file. +/// +/// This is reference-stable rather than a delta: the fragment id is committed +/// and physical row offsets never move, so the post-image is unambiguous. The +/// newly-deleted rows -- the delta rebase and conflict detection need -- are +/// derived by diffing against the read version rather than serialized. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct SetDeletionFile { + /// The fragment, by committed id. Unlike its sibling fragment actions this + /// takes no [`Ref`]: a fragment minted in the same operation has no + /// committed rows to delete. + pub fragment: u64, + /// The new deletion file, or `None` to clear the fragment's deletions. + pub deletion_file: Option, + /// See [`AddFragment::data_change`]. + pub data_change: bool, +} + +/// Alter facets of an existing field in place, preserving its id. +/// +/// Each facet is independently optional -- present means "change this", absent +/// means "leave it alone" -- so a widening cast and a nullability relaxation on +/// the same field commute. A cast additionally needs a [`TombstoneFieldData`] +/// plus a fresh [`AddDataFile`] to rewrite the data. +#[derive(Debug, Clone, PartialEq, DeepSizeOf, Default)] +pub struct AlterField { + pub field: i32, + pub name: Option, + /// The new Arrow logical type. The cast. + pub logical_type: Option, + pub nullable: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ref_accessors() { + assert_eq!(Ref::Committed(7).committed(), Some(7)); + assert_eq!(Ref::Committed(7).local(), None); + assert_eq!(Ref::Local(2).local(), Some(2)); + assert_eq!(Ref::Local(2).committed(), None); + } + + #[test] + fn test_data_change_defaults_by_action_kind() { + let alter = Action::AlterField(AlterField { + field: 1, + name: Some("renamed".into()), + ..Default::default() + }); + assert!(!alter.is_data_change(), "a rename changes no row values"); + + let remove = Action::RemoveFragment(RemoveFragment { + fragment: Ref::Committed(3), + data_change: true, + }); + assert!(remove.is_data_change()); + } + + #[test] + fn test_iter_actions_flattens_steps_in_order() { + let operation = UserOperation::new( + "two steps", + vec![ + UserAction::new( + "first", + vec![Action::RemoveFragment(RemoveFragment { + fragment: Ref::Committed(1), + data_change: true, + })], + ), + UserAction::new( + "second", + vec![Action::RemoveFragment(RemoveFragment { + fragment: Ref::Committed(2), + data_change: true, + })], + ), + ], + ); + let fragments = operation + .iter_actions() + .map(|action| match action { + Action::RemoveFragment(remove) => remove.fragment, + other => panic!("unexpected action {other}"), + }) + .collect::>(); + assert_eq!(fragments, vec![Ref::Committed(1), Ref::Committed(2)]); + } +} From b00cf9f9fe3c4c9e767f84d559f44927f940f690 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sat, 15 Aug 2026 17:05:18 -0700 Subject: [PATCH 02/24] feat(transaction): carry action-based operations through the wire Adds `Operation::UserOperation`, the Transaction V2 arm of the transaction oneof, and its protobuf conversions in both directions. Loading a V2 transaction now yields an action set instead of an outright rejection; an action the build does not implement is still rejected rather than skipped, so a concurrent V2 commit can never be silently treated as a no-op. The rest of the transaction machinery gains the variant but no behavior: build_manifest returns NotSupported, and conflict checks route through a single `check_action_txn` that conservatively demands a retry. Apply and conflict rules follow in later commits. Also extracts `From<&DeletionFile> for pb::DeletionFile`, previously inlined in the fragment conversion and now needed by SetDeletionFile too. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance-table/src/format/fragment.rs | 30 +- rust/lance-table/src/transaction/action.rs | 3 + .../src/transaction/action/proto.rs | 577 ++++++++++++++++++ rust/lance-table/src/transaction/conflicts.rs | 7 + .../src/transaction/manifest_build.rs | 5 + rust/lance-table/src/transaction/operation.rs | 11 + rust/lance-table/src/transaction/proto.rs | 89 ++- rust/lance/src/io/commit/conflict_resolver.rs | 86 ++- 8 files changed, 768 insertions(+), 40 deletions(-) create mode 100644 rust/lance-table/src/transaction/action/proto.rs diff --git a/rust/lance-table/src/format/fragment.rs b/rust/lance-table/src/format/fragment.rs index 149a9b44fc6..bf49d742fee 100644 --- a/rust/lance-table/src/format/fragment.rs +++ b/rust/lance-table/src/format/fragment.rs @@ -455,6 +455,22 @@ impl TryFrom for DeletionFile { } } +impl From<&DeletionFile> for pb::DeletionFile { + fn from(value: &DeletionFile) -> Self { + let file_type = match value.file_type { + DeletionFileType::Array => pb::deletion_file::DeletionFileType::ArrowArray, + DeletionFileType::Bitmap => pb::deletion_file::DeletionFileType::Bitmap, + }; + Self { + read_version: value.read_version, + id: value.id, + file_type: file_type.into(), + num_deleted_rows: value.num_deleted_rows.unwrap_or_default() as u64, + base_id: value.base_id, + } + } +} + /// Data fragment. /// /// A fragment is a set of files which represent the different columns of the same rows. @@ -716,19 +732,7 @@ impl TryFrom for Fragment { impl From<&Fragment> for pb::DataFragment { fn from(f: &Fragment) -> Self { - let deletion_file = f.deletion_file.as_ref().map(|f| { - let file_type = match f.file_type { - DeletionFileType::Array => pb::deletion_file::DeletionFileType::ArrowArray, - DeletionFileType::Bitmap => pb::deletion_file::DeletionFileType::Bitmap, - }; - pb::DeletionFile { - read_version: f.read_version, - id: f.id, - file_type: file_type.into(), - num_deleted_rows: f.num_deleted_rows.unwrap_or_default() as u64, - base_id: f.base_id, - } - }); + let deletion_file = f.deletion_file.as_ref().map(pb::DeletionFile::from); let row_id_sequence = f.row_id_meta.as_ref().map(|m| match m { RowIdMeta::Inline(data) => { diff --git a/rust/lance-table/src/transaction/action.rs b/rust/lance-table/src/transaction/action.rs index 28137675b69..f831044c708 100644 --- a/rust/lance-table/src/transaction/action.rs +++ b/rust/lance-table/src/transaction/action.rs @@ -16,6 +16,7 @@ //! //! ```text //! action the vocabulary (this module) +//! action::proto its persisted protobuf encoding //! ``` //! //! # Stability @@ -24,6 +25,8 @@ //! contract, and a transaction carrying a [`UserOperation`] is rejected outright //! by libraries that predate it. +mod proto; + use crate::format::{BasePath, DataFile, DeletionFile, RowIdMeta}; use crate::rowids::version::RowDatasetVersionMeta; use lance_core::datatypes::Field; diff --git a/rust/lance-table/src/transaction/action/proto.rs b/rust/lance-table/src/transaction/action/proto.rs new file mode 100644 index 00000000000..4dd4d320711 --- /dev/null +++ b/rust/lance-table/src/transaction/action/proto.rs @@ -0,0 +1,577 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Conversions between the action vocabulary and its protobuf encoding. +//! +//! Reading is fail-closed: an action this build does not implement is an error, +//! never a silently skipped element. The commit path collects concurrent +//! transactions with `try_collect`, so a transaction carrying an unknown action +//! must abort the commit rather than be treated as a no-op. + +use super::{ + Action, AddBase, AddDataFile, AddField, AddFragment, AlterField, Ref, RemoveFragment, + SetDeletionFile, TombstoneFieldData, UserAction, UserOperation, +}; +use crate::format::pb; +use crate::format::{BasePath, DataFile, DeletionFile, ExternalFile, RowIdMeta}; +use crate::rowids::version::RowDatasetVersionMeta; +use lance_core::datatypes::Field; +use lance_core::{Error, Result}; + +/// A field id on the wire is a `uint64`; in the manifest it is an `i32`. +fn field_id_from_wire(id: u64) -> Result { + i32::try_from(id).map_err(|_| { + Error::invalid_input(format!( + "field id {id} in an action exceeds the maximum field id ({})", + i32::MAX + )) + }) +} + +impl From for pb::Ref { + fn from(value: Ref) -> Self { + let kind = match value { + Ref::Committed(id) => pb::r#ref::Kind::Committed(id), + Ref::Local(token) => pb::r#ref::Kind::Local(token), + }; + Self { kind: Some(kind) } + } +} + +impl TryFrom for Ref { + type Error = Error; + + fn try_from(message: pb::Ref) -> Result { + match message.kind { + Some(pb::r#ref::Kind::Committed(id)) => Ok(Self::Committed(id)), + Some(pb::r#ref::Kind::Local(token)) => Ok(Self::Local(token)), + None => Err(Error::invalid_input( + "a Ref in an action was empty; it must be either committed or local", + )), + } + } +} + +/// `data_change` is absent-means-true on the wire, so only the `false` case is +/// written out. +fn data_change_to_wire(data_change: bool) -> Option { + (!data_change).then_some(false) +} + +fn data_change_from_wire(data_change: Option) -> bool { + data_change.unwrap_or(true) +} + +impl From<&UserOperation> for pb::UserOperation { + fn from(value: &UserOperation) -> Self { + Self { + description: value.description.clone(), + // uuid and read_version mirror the enclosing Transaction and are + // stamped in by its conversion. + uuid: String::new(), + read_version: 0, + actions: value.actions.iter().map(pb::UserAction::from).collect(), + } + } +} + +impl TryFrom for UserOperation { + type Error = Error; + + fn try_from(message: pb::UserOperation) -> Result { + Ok(Self { + description: message.description, + actions: message + .actions + .into_iter() + .map(UserAction::try_from) + .collect::>>()?, + }) + } +} + +impl From<&UserAction> for pb::UserAction { + fn from(value: &UserAction) -> Self { + Self { + description: value.description.clone(), + actions: value.actions.iter().map(pb::Action::from).collect(), + } + } +} + +impl TryFrom for UserAction { + type Error = Error; + + fn try_from(message: pb::UserAction) -> Result { + Ok(Self { + description: message.description, + actions: message + .actions + .into_iter() + .map(Action::try_from) + .collect::>>()?, + }) + } +} + +impl From<&Action> for pb::Action { + fn from(value: &Action) -> Self { + let action = match value { + Action::AddFragment(action) => pb::action::Action::AddFragment(action.into()), + Action::AddDataFile(action) => pb::action::Action::AddDataFile(action.into()), + Action::AddField(action) => pb::action::Action::AddField(action.into()), + Action::AddBase(action) => pb::action::Action::AddBase(action.into()), + Action::TombstoneFieldData(action) => { + pb::action::Action::TombstoneFieldData(action.into()) + } + Action::RemoveFragment(action) => pb::action::Action::RemoveFragment(action.into()), + Action::SetDeletionFile(action) => pb::action::Action::SetDeletionFile(action.into()), + Action::AlterField(action) => pb::action::Action::AlterField(action.into()), + }; + Self { + action: Some(action), + } + } +} + +impl TryFrom for Action { + type Error = Error; + + fn try_from(message: pb::Action) -> Result { + match message.action { + Some(pb::action::Action::AddFragment(action)) => { + Ok(Self::AddFragment(action.try_into()?)) + } + Some(pb::action::Action::AddDataFile(action)) => { + Ok(Self::AddDataFile(action.try_into()?)) + } + Some(pb::action::Action::AddField(action)) => Ok(Self::AddField(action.try_into()?)), + Some(pb::action::Action::AddBase(action)) => Ok(Self::AddBase(action.try_into()?)), + Some(pb::action::Action::TombstoneFieldData(action)) => { + Ok(Self::TombstoneFieldData(action.try_into()?)) + } + Some(pb::action::Action::RemoveFragment(action)) => { + Ok(Self::RemoveFragment(action.try_into()?)) + } + Some(pb::action::Action::SetDeletionFile(action)) => { + Ok(Self::SetDeletionFile(action.try_into()?)) + } + Some(pb::action::Action::AlterField(action)) => { + Ok(Self::AlterField(action.try_into()?)) + } + // The drafted vocabulary is larger than what is implemented. Reject + // rather than skip: silently dropping an action would apply a + // partial transaction. + Some(other) => Err(Error::not_supported(format!( + "the action-based transaction uses action {other:?}, which is drafted but not \ + implemented by this version of Lance", + ))), + None => Err(Error::invalid_input( + "an Action in a user operation was empty", + )), + } + } +} + +impl From<&AddFragment> for pb::AddFragment { + fn from(value: &AddFragment) -> Self { + Self { + local: value.local, + physical_rows: value.physical_rows, + row_id_sequence: value.row_id_meta.as_ref().map(|meta| match meta { + RowIdMeta::Inline(data) => { + pb::add_fragment::RowIdSequence::InlineRowIds(data.clone()) + } + RowIdMeta::External(file) => { + pb::add_fragment::RowIdSequence::ExternalRowIds(external_file_to_wire(file)) + } + }), + last_updated_at_version_sequence: value + .last_updated_at_version_meta + .as_ref() + .map(|meta| match meta { + RowDatasetVersionMeta::Inline(data) => { + pb::add_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions( + data.to_vec(), + ) + } + RowDatasetVersionMeta::External(file) => { + pb::add_fragment::LastUpdatedAtVersionSequence::ExternalLastUpdatedAtVersions( + external_file_to_wire(file), + ) + } + }), + created_at_version_sequence: value + .created_at_version_meta + .as_ref() + .map(|meta| match meta { + RowDatasetVersionMeta::Inline(data) => { + pb::add_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions( + data.to_vec(), + ) + } + RowDatasetVersionMeta::External(file) => { + pb::add_fragment::CreatedAtVersionSequence::ExternalCreatedAtVersions( + external_file_to_wire(file), + ) + } + }), + data_change: data_change_to_wire(value.data_change), + } + } +} + +impl TryFrom for AddFragment { + type Error = Error; + + fn try_from(message: pb::AddFragment) -> Result { + Ok(Self { + local: message.local, + physical_rows: message.physical_rows, + row_id_meta: message.row_id_sequence.map(|sequence| match sequence { + pb::add_fragment::RowIdSequence::InlineRowIds(data) => RowIdMeta::Inline(data), + pb::add_fragment::RowIdSequence::ExternalRowIds(file) => { + RowIdMeta::External(external_file_from_wire(file)) + } + }), + last_updated_at_version_meta: message.last_updated_at_version_sequence.map( + |sequence| { + match sequence { + pb::add_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions( + data, + ) => RowDatasetVersionMeta::Inline(data.into()), + pb::add_fragment::LastUpdatedAtVersionSequence::ExternalLastUpdatedAtVersions( + file, + ) => RowDatasetVersionMeta::External(external_file_from_wire(file)), + } + }, + ), + created_at_version_meta: message.created_at_version_sequence.map(|sequence| { + match sequence { + pb::add_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions(data) => { + RowDatasetVersionMeta::Inline(data.into()) + } + pb::add_fragment::CreatedAtVersionSequence::ExternalCreatedAtVersions(file) => { + RowDatasetVersionMeta::External(external_file_from_wire(file)) + } + } + }), + data_change: data_change_from_wire(message.data_change), + }) + } +} + +fn external_file_to_wire(file: &ExternalFile) -> pb::ExternalFile { + pb::ExternalFile { + path: file.path.clone(), + offset: file.offset, + size: file.size, + } +} + +fn external_file_from_wire(file: pb::ExternalFile) -> ExternalFile { + ExternalFile { + path: file.path, + offset: file.offset, + size: file.size, + } +} + +impl From<&AddDataFile> for pb::AddDataFile { + fn from(value: &AddDataFile) -> Self { + Self { + fragment: Some(value.fragment.into()), + file: Some(pb::DataFile::from(&value.file)), + field_ids: value.field_ids.iter().map(|id| (*id).into()).collect(), + data_change: data_change_to_wire(value.data_change), + } + } +} + +impl TryFrom for AddDataFile { + type Error = Error; + + fn try_from(message: pb::AddDataFile) -> Result { + Ok(Self { + fragment: required(message.fragment, "AddDataFile.fragment")?.try_into()?, + file: DataFile::try_from(required(message.file, "AddDataFile.file")?)?, + field_ids: message + .field_ids + .into_iter() + .map(Ref::try_from) + .collect::>>()?, + data_change: data_change_from_wire(message.data_change), + }) + } +} + +impl From<&AddField> for pb::AddField { + fn from(value: &AddField) -> Self { + Self { + local: value.local, + parent: value.parent.map(Into::into), + def: Some(lance_file::format::pb::Field::from(&value.def)), + } + } +} + +impl TryFrom for AddField { + type Error = Error; + + fn try_from(message: pb::AddField) -> Result { + Ok(Self { + local: message.local, + parent: message.parent.map(Ref::try_from).transpose()?, + def: Field::from(&required(message.def, "AddField.def")?), + }) + } +} + +impl From<&AddBase> for pb::AddBase { + fn from(value: &AddBase) -> Self { + Self { + local: value.local, + base: Some(pb::BasePath::from(value.base.clone())), + } + } +} + +impl TryFrom for AddBase { + type Error = Error; + + fn try_from(message: pb::AddBase) -> Result { + Ok(Self { + local: message.local, + base: BasePath::from(required(message.base, "AddBase.base")?), + }) + } +} + +impl From<&TombstoneFieldData> for pb::TombstoneFieldData { + fn from(value: &TombstoneFieldData) -> Self { + Self { + fragment: Some(value.fragment.into()), + field_ids: value.field_ids.iter().map(|id| *id as u64).collect(), + data_change: data_change_to_wire(value.data_change), + } + } +} + +impl TryFrom for TombstoneFieldData { + type Error = Error; + + fn try_from(message: pb::TombstoneFieldData) -> Result { + Ok(Self { + fragment: required(message.fragment, "TombstoneFieldData.fragment")?.try_into()?, + field_ids: message + .field_ids + .into_iter() + .map(field_id_from_wire) + .collect::>>()?, + data_change: data_change_from_wire(message.data_change), + }) + } +} + +impl From<&RemoveFragment> for pb::RemoveFragment { + fn from(value: &RemoveFragment) -> Self { + Self { + fragment: Some(value.fragment.into()), + data_change: data_change_to_wire(value.data_change), + } + } +} + +impl TryFrom for RemoveFragment { + type Error = Error; + + fn try_from(message: pb::RemoveFragment) -> Result { + Ok(Self { + fragment: required(message.fragment, "RemoveFragment.fragment")?.try_into()?, + data_change: data_change_from_wire(message.data_change), + }) + } +} + +impl From<&SetDeletionFile> for pb::SetDeletionFile { + fn from(value: &SetDeletionFile) -> Self { + Self { + fragment: value.fragment, + deletion_file: value.deletion_file.as_ref().map(pb::DeletionFile::from), + data_change: data_change_to_wire(value.data_change), + } + } +} + +impl TryFrom for SetDeletionFile { + type Error = Error; + + fn try_from(message: pb::SetDeletionFile) -> Result { + Ok(Self { + fragment: message.fragment, + deletion_file: message + .deletion_file + .map(DeletionFile::try_from) + .transpose()?, + data_change: data_change_from_wire(message.data_change), + }) + } +} + +impl From<&AlterField> for pb::AlterField { + fn from(value: &AlterField) -> Self { + Self { + field: value.field as u64, + name: value.name.clone(), + logical_type: value.logical_type.clone(), + nullable: value.nullable, + } + } +} + +impl TryFrom for AlterField { + type Error = Error; + + fn try_from(message: pb::AlterField) -> Result { + Ok(Self { + field: field_id_from_wire(message.field)?, + name: message.name, + logical_type: message.logical_type, + nullable: message.nullable, + }) + } +} + +fn required(value: Option, what: &str) -> Result { + value.ok_or_else(|| Error::invalid_input(format!("{what} is required but was not set"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::{DeletionFileType, pb}; + use arrow_schema::{DataType, Field as ArrowField}; + use std::sync::Arc; + + fn sample_data_file() -> DataFile { + DataFile::new_unstarted("data/1.lance", 2, 0) + } + + fn all_actions() -> Vec { + vec![ + Action::AddFragment(AddFragment { + local: 0, + physical_rows: 10, + row_id_meta: Some(RowIdMeta::Inline(vec![1, 2, 3])), + last_updated_at_version_meta: Some(RowDatasetVersionMeta::Inline(Arc::from( + [4u8, 5].as_slice(), + ))), + created_at_version_meta: None, + data_change: true, + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Local(0), + file: sample_data_file(), + field_ids: vec![Ref::Committed(1), Ref::Local(3)], + data_change: false, + }), + Action::AddField(AddField { + local: 3, + parent: Some(Ref::Committed(1)), + def: Field::try_from(ArrowField::new("added", DataType::Int32, true)).unwrap(), + }), + Action::AddBase(AddBase { + local: 1, + base: BasePath::new(0, "s3://bucket/x".into(), Some("other".into()), false), + }), + Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Committed(4), + field_ids: vec![7, 8], + data_change: true, + }), + Action::RemoveFragment(RemoveFragment { + fragment: Ref::Committed(5), + data_change: true, + }), + Action::SetDeletionFile(SetDeletionFile { + fragment: 6, + deletion_file: Some(DeletionFile { + read_version: 3, + id: 9, + file_type: DeletionFileType::Bitmap, + num_deleted_rows: Some(4), + base_id: None, + }), + data_change: true, + }), + Action::AlterField(AlterField { + field: 2, + name: Some("renamed".into()), + logical_type: Some("int64".into()), + nullable: Some(false), + }), + ] + } + + #[test] + fn test_user_operation_round_trips() { + let operation = UserOperation::new( + "compound commit", + vec![ + UserAction::new("everything", all_actions()), + UserAction::new("nothing", vec![]), + ], + ); + + let message = pb::UserOperation::from(&operation); + let round_tripped = UserOperation::try_from(message).unwrap(); + + assert_eq!(round_tripped, operation); + } + + #[test] + fn test_data_change_is_absent_when_true() { + // Absent means "real change" on the wire, so the common case costs no + // bytes and an old field-less writer is read correctly. + let message = pb::RemoveFragment::from(&RemoveFragment { + fragment: Ref::Committed(1), + data_change: true, + }); + assert_eq!(message.data_change, None); + assert!(RemoveFragment::try_from(message).unwrap().data_change); + } + + #[test] + fn test_unimplemented_action_is_rejected() { + let message = pb::Action { + action: Some(pb::action::Action::ResetTable(pb::ResetTable {})), + }; + let error = Action::try_from(message).unwrap_err(); + assert!( + matches!(error, Error::NotSupported { .. }), + "expected NotSupported, got {error:?}" + ); + assert!( + error.to_string().contains("not implemented"), + "unexpected message: {error}" + ); + } + + #[test] + fn test_empty_action_is_rejected() { + let error = Action::try_from(pb::Action { action: None }).unwrap_err(); + assert!( + error.to_string().contains("was empty"), + "unexpected message: {error}" + ); + } + + #[test] + fn test_empty_ref_is_rejected() { + let error = Ref::try_from(pb::Ref { kind: None }).unwrap_err(); + assert!( + error.to_string().contains("committed or local"), + "unexpected message: {error}" + ); + } +} diff --git a/rust/lance-table/src/transaction/conflicts.rs b/rust/lance-table/src/transaction/conflicts.rs index ad942d6c182..57f384e24fc 100644 --- a/rust/lance-table/src/transaction/conflicts.rs +++ b/rust/lance-table/src/transaction/conflicts.rs @@ -863,6 +863,13 @@ impl PartialEq for Operation { std::mem::discriminant(self) == std::mem::discriminant(other) } (Self::DataOverlay { groups: a }, Self::DataOverlay { groups: b }) => compare_vec(a, b), + // A V2 operation is an ordered list, so unlike the operations above + // it compares element-wise with no order-insensitivity to work + // around. It is never equal to a legacy operation: equality here + // answers "is the operation I am holding the one already + // committed?", and a translated operation is a different commit. + (Self::UserOperation(a), Self::UserOperation(b)) => a == b, + (Self::UserOperation(_), _) | (_, Self::UserOperation(_)) => false, (Self::DataOverlay { .. }, _) | (_, Self::DataOverlay { .. }) => false, } } diff --git a/rust/lance-table/src/transaction/manifest_build.rs b/rust/lance-table/src/transaction/manifest_build.rs index e47bc139e58..3d673a5e2ec 100644 --- a/rust/lance-table/src/transaction/manifest_build.rs +++ b/rust/lance-table/src/transaction/manifest_build.rs @@ -1287,6 +1287,11 @@ impl Transaction { // Base paths are handled in the manifest creation section below final_fragments.extend(maybe_existing_fragments?.clone()); } + Operation::UserOperation(_) => { + return Err(Error::not_supported( + "applying an action-based transaction is not implemented yet", + )); + } }; // If a fragment was reserved then it may not belong at the end of the fragments list. diff --git a/rust/lance-table/src/transaction/operation.rs b/rust/lance-table/src/transaction/operation.rs index 1874984864b..a8318187aad 100644 --- a/rust/lance-table/src/transaction/operation.rs +++ b/rust/lance-table/src/transaction/operation.rs @@ -14,6 +14,7 @@ use crate::format::overlay::DataOverlayFile; use crate::format::{BasePath, DataFile, Fragment, IndexFile, IndexMetadata}; use crate::system_index::mem_wal::CompactedSsTable; use crate::transaction::UpdateMap; +use crate::transaction::action::UserOperation; use lance_core::datatypes::Schema; use lance_core::deepsize::DeepSizeOf; use roaring::RoaringBitmap; @@ -220,6 +221,14 @@ pub enum Operation { /// The new base paths to add to the manifest. new_bases: Vec, }, + + /// A Transaction V2 operation: an ordered list of granular actions that + /// commit atomically as one manifest change. + /// + /// Unlike the variants above, this one is not a single named change -- it is + /// the composable form the others decompose into. See + /// [`super::action`] for the vocabulary and its stability caveats. + UserOperation(UserOperation), } #[derive(Debug, Clone, PartialEq, DeepSizeOf)] @@ -270,6 +279,7 @@ impl std::fmt::Display for Operation { Self::Clone { .. } => write!(f, "Clone"), Self::UpdateMemWalState { .. } => write!(f, "UpdateMemWalState"), Self::UpdateBases { .. } => write!(f, "UpdateBases"), + Self::UserOperation(_) => write!(f, "UserOperation"), } } } @@ -329,6 +339,7 @@ impl Operation { Self::UpdateMemWalState { .. } => "UpdateMemWalState", Self::Clone { .. } => "Clone", Self::UpdateBases { .. } => "UpdateBases", + Self::UserOperation(_) => "UserOperation", } } } diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs index 5082ec8b57e..03148ffd1e6 100644 --- a/rust/lance-table/src/transaction/proto.rs +++ b/rust/lance-table/src/transaction/proto.rs @@ -12,6 +12,7 @@ use crate::format::key_existence::KeyExistenceFilter; use crate::format::pb; use crate::format::{BasePath, Fragment, IndexFile, IndexMetadata, overlay::DataOverlayFile}; use crate::system_index::mem_wal::CompactedSsTable; +use crate::transaction::action::UserOperation; use crate::transaction::{ DataOverlayGroup, DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, Transaction, UpdateMap, UpdateMapEntry, UpdateMode, UpdatedFragmentOffsets, translate_config_updates, @@ -416,18 +417,14 @@ impl TryFrom for Transaction { .map(DataOverlayGroup::try_from) .collect::>>()?, }, - Some(pb::transaction::Operation::UserOperation(_)) => { + Some(pb::transaction::Operation::UserOperation(user_operation)) => { // Action-based transactions (Transaction V2) are a draft wire - // format (OSS-1530). This version of Lance recognizes the message - // but has no support for it: reject on load, fail-closed. Because - // load_and_sort_new_transactions collects transactions with - // try_collect, a concurrent V2 commit in the conflict window - // aborts the whole commit rather than being silently skipped. - // Do NOT make this parsing lenient. - return Err(Error::not_supported( - "action-based transactions (Transaction V2) are not supported \ - by this version of Lance; please upgrade", - )); + // format (OSS-1530). Parsing is fail-closed: an action this build + // does not implement is an error, never a skipped element. + // load_and_sort_new_transactions collects concurrent transactions + // with try_collect, so such a transaction aborts the commit rather + // than being silently treated as a no-op. Do NOT make this lenient. + Operation::UserOperation(UserOperation::try_from(user_operation)?) } None => { return Err(Error::internal( @@ -732,6 +729,15 @@ impl From<&Transaction> for pb::Transaction { .collect::>(), }) } + Operation::UserOperation(user_operation) => { + let mut message = pb::UserOperation::from(user_operation); + // The operation's identity and read version are the enclosing + // transaction's; the wire carries them in both places so a + // squashed operation keeps its own provenance. + message.uuid = value.uuid.clone(); + message.read_version = value.read_version; + pb::transaction::Operation::UserOperation(message) + } }; let transaction_properties = value @@ -836,6 +842,7 @@ mod tests { use super::*; use crate::format::DataFile; use crate::format::overlay::OverlayCoverage; + use crate::transaction::action::{Action, AddFragment, UserAction}; #[test] fn test_data_overlay_operation_roundtrips() { @@ -882,28 +889,62 @@ mod tests { } #[test] - fn test_user_operation_rejected_on_load() { - // Action-based transactions (Transaction V2) are a draft wire format that - // this version of Lance does not support. Loading one must fail closed - // (never be silently skipped or leniently parsed), so that a concurrent - // V2 commit in the conflict window aborts an in-flight commit. + fn test_user_operation_round_trips_through_transaction() { + let uuid = Uuid::new_v4().to_string(); + let transaction = Transaction { + read_version: 4, + uuid: uuid.clone(), + operation: Operation::UserOperation(UserOperation::new( + "INSERT INTO t VALUES (1)", + vec![UserAction::new( + "append batch", + vec![Action::AddFragment(AddFragment { + local: 0, + physical_rows: 1, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + })], + )], + )), + tag: None, + transaction_properties: None, + }; + + let message = pb::Transaction::from(&transaction); + // The operation repeats the envelope's identity so a squashed operation + // keeps the provenance of the commit it came from. + match &message.operation { + Some(pb::transaction::Operation::UserOperation(user_operation)) => { + assert_eq!(user_operation.uuid, uuid); + assert_eq!(user_operation.read_version, 4); + } + other => panic!("expected UserOperation, got {other:?}"), + } + + assert_eq!(Transaction::try_from(message).unwrap(), transaction); + } + + #[test] + fn test_unimplemented_action_fails_closed_on_load() { + // The drafted vocabulary is larger than what is implemented. Loading a + // transaction that uses an unimplemented action must fail rather than + // parse leniently: load_and_sort_new_transactions collects concurrent + // transactions with try_collect, so this aborts an in-flight commit + // instead of letting it proceed against a change it cannot see. let message = pb::Transaction { read_version: 1, uuid: Uuid::new_v4().to_string(), operation: Some(pb::transaction::Operation::UserOperation( pb::UserOperation { - description: "INSERT INTO t VALUES (1)".to_string(), + description: "DROP TABLE t".to_string(), uuid: Uuid::new_v4().to_string(), read_version: 1, actions: vec![pb::UserAction { - description: "append batch".to_string(), + description: "reset".to_string(), actions: vec![pb::Action { - action: Some(pb::action::Action::AddFragment(pb::AddFragment { - local: 0, - physical_rows: 1, - data_change: Some(true), - ..Default::default() - })), + action: Some(pb::action::Action::ResetTable(pb::ResetTable {})), }], }], }, diff --git a/rust/lance/src/io/commit/conflict_resolver.rs b/rust/lance/src/io/commit/conflict_resolver.rs index 99b5188f91b..3a0f4fbb6e8 100644 --- a/rust/lance/src/io/commit/conflict_resolver.rs +++ b/rust/lance/src/io/commit/conflict_resolver.rs @@ -89,7 +89,11 @@ impl<'a> TransactionRebase<'a> { | Operation::UpdateMemWalState { .. } | Operation::Clone { .. } | Operation::Restore { .. } - | Operation::UpdateBases { .. } => Ok(Self { + | Operation::UpdateBases { .. } + // An action set can modify fragments, but check_action_txn rejects + // any concurrency before the rebase state is consulted, so there is + // nothing to collect yet. + | Operation::UserOperation(_) => Ok(Self { transaction, affected_rows, initial_fragments: HashMap::new(), @@ -314,9 +318,24 @@ impl<'a> TransactionRebase<'a> { Operation::UpdateBases { .. } => { self.check_add_bases_txn(other_transaction, other_version) } + Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), } } + /// Whether an action-based transaction conflicts with `other_transaction`. + /// + /// Reached from both directions: when the transaction being committed is + /// action-based, and when a concurrent one is. + fn check_action_txn( + &mut self, + other_transaction: &Transaction, + other_version: u64, + ) -> Result<()> { + // Conservative until action footprints land: an action-based + // transaction on either side means retry against the newer version. + Err(self.retryable_conflict_err(other_transaction, other_version)) + } + fn check_delete_txn( &mut self, other_transaction: &Transaction, @@ -324,6 +343,11 @@ impl<'a> TransactionRebase<'a> { ) -> Result<()> { if let Operation::Delete { .. } = &self.transaction.operation { match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } Operation::CreateIndex { .. } | Operation::ReserveFragments { .. } | Operation::Clone { .. } @@ -478,6 +502,11 @@ impl<'a> TransactionRebase<'a> { } match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } Operation::CreateIndex { .. } | Operation::ReserveFragments { .. } | Operation::Project { .. } @@ -641,6 +670,11 @@ impl<'a> TransactionRebase<'a> { } = &mut self.transaction.operation { match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } Operation::Append { .. } | Operation::Clone { .. } // An overlay committed after this index's version is newer than @@ -840,6 +874,11 @@ impl<'a> TransactionRebase<'a> { } = &self.transaction.operation { match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } // Rewrite is only compatible with operations that don't touch // existing fragments or update fragments we don't touch. Operation::Append { .. } @@ -1030,6 +1069,9 @@ impl<'a> TransactionRebase<'a> { other_version: u64, ) -> Result<()> { match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), Operation::Overwrite { .. } => { if self .transaction @@ -1079,6 +1121,9 @@ impl<'a> TransactionRebase<'a> { other_version: u64, ) -> Result<()> { match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), // Append is not compatible with any operation that completely // overwrites the schema. Operation::Overwrite { .. } @@ -1109,6 +1154,11 @@ impl<'a> TransactionRebase<'a> { ) -> Result<()> { if let Operation::DataReplacement { replacements } = &self.transaction.operation { match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } Operation::Append { .. } | Operation::Clone { .. } | Operation::UpdateConfig { .. } @@ -1287,6 +1337,9 @@ impl<'a> TransactionRebase<'a> { other_version: u64, ) -> Result<()> { match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), Operation::Append { .. } | Operation::CreateIndex { .. } | Operation::ReserveFragments { .. } @@ -1385,6 +1438,9 @@ impl<'a> TransactionRebase<'a> { other_version: u64, ) -> Result<()> { match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), // See the MemWAL exception in check_create_index_txn. Operation::CreateIndex { new_indices, .. } => { if new_indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME) { @@ -1422,6 +1478,9 @@ impl<'a> TransactionRebase<'a> { other_version: u64, ) -> Result<()> { match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), Operation::Append { .. } | Operation::Delete { .. } | Operation::Overwrite { .. } @@ -1449,6 +1508,9 @@ impl<'a> TransactionRebase<'a> { other_version: u64, ) -> Result<()> { match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), Operation::Overwrite { .. } | Operation::Restore { .. } => { Err(self.incompatible_conflict_err(other_transaction, other_version)) } @@ -1475,6 +1537,9 @@ impl<'a> TransactionRebase<'a> { other_version: u64, ) -> Result<()> { match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), // Project is compatible with anything that doesn't change the schema Operation::Append { .. } | Operation::Update { .. } @@ -1511,6 +1576,11 @@ impl<'a> TransactionRebase<'a> { } = &self.transaction.operation { match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } Operation::Overwrite { .. } => { // Updates to schema metadata or field metadata conflict with any kind // of overwrite. @@ -1574,6 +1644,11 @@ impl<'a> TransactionRebase<'a> { } = &self.transaction.operation { match &other_transaction.operation { + // A concurrent action-based transaction is compared by action + // footprint rather than by operation pair. + Operation::UserOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } Operation::UpdateMemWalState { compacted_sstables: other_compacted_sstables, .. @@ -1727,7 +1802,11 @@ impl<'a> TransactionRebase<'a> { | Operation::Clone { .. } | Operation::UpdateConfig { .. } | Operation::UpdateMemWalState { .. } - | Operation::UpdateBases { .. } => Ok(self.transaction), + | Operation::UpdateBases { .. } + // Rebasing an action set (relocating its minted ids onto the newer + // version) is not implemented yet; check_action_txn rejects before + // this is reached. + | Operation::UserOperation(_) => Ok(self.transaction), } } @@ -4468,7 +4547,8 @@ mod tests { | Operation::UpdateConfig { .. } | Operation::UpdateBases { .. } | Operation::Restore { .. } - | Operation::UpdateMemWalState { .. } => Box::new(std::iter::empty()), + | Operation::UpdateMemWalState { .. } + | Operation::UserOperation(_) => Box::new(std::iter::empty()), Operation::Delete { updated_fragments, deleted_fragment_ids, From 466286d6bb0c0bdb1b9f98270e00af04e6ce3610 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sat, 15 Aug 2026 17:07:15 -0700 Subject: [PATCH 03/24] refactor(table): extract manifest assembly helpers from build_manifest Pulls the two operation-independent stages out of `build_manifest`: `normalize_fragments` (order, drop fully-tombstoned files, check overlay order) and `assemble_manifest` (construct the manifest and apply the tag, feature flags, timestamp, and fragment id watermark). The overwrite-only storage format override becomes an explicit parameter rather than a match on the operation inside the assembly step. No behavior change; the action-based apply path needs the same two stages. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/transaction/manifest_build.rs | 185 +++++++++++------- 1 file changed, 113 insertions(+), 72 deletions(-) diff --git a/rust/lance-table/src/transaction/manifest_build.rs b/rust/lance-table/src/transaction/manifest_build.rs index 3d673a5e2ec..acc61ab5090 100644 --- a/rust/lance-table/src/transaction/manifest_build.rs +++ b/rust/lance-table/src/transaction/manifest_build.rs @@ -1294,27 +1294,12 @@ impl Transaction { } }; - // If a fragment was reserved then it may not belong at the end of the fragments list. - final_fragments.sort_by_key(|frag| frag.id); + Self::normalize_fragments(&mut final_fragments)?; - // Clean up data files that only contain tombstoned fields - Self::remove_tombstoned_data_files(&mut final_fragments); - - // Enforce the newest-last overlay ordering invariant at the write - // boundary. Load normalizes with a sort; this rejects any commit path - // that assembled a fragment's overlays out of order. - for fragment in &final_fragments { - if !fragment.overlays.is_empty() { - crate::format::overlay::verify_overlays_newest_last(&fragment.overlays)?; - } - } - - let user_requested_version = match (&config.storage_format, config.use_legacy_format) { - (Some(storage_format), _) => Some(storage_format.lance_file_format()), - (None, Some(true)) => Some(ConcreteFileVersion::V1), - (None, Some(false)) => Some(ConcreteFileVersion::V2_0), - (None, None) => None, - }; + // If this is an overwrite operation and the user has requested a specific + // version then overwrite with that version. Otherwise, if the user didn't + // request a specific version, then keep whatever version we had before. + let overwrite_storage_format = matches!(self.operation, Operation::Overwrite { .. }); // Applied once the final index list is known, so it sees exactly the // indices this commit publishes rather than what any one operation arm @@ -1330,55 +1315,14 @@ impl Transaction { )?; } - let mut manifest = if let Some(current_manifest) = current_manifest { - // OVERWRITE with initial_bases on existing dataset is not allowed (caught by validation) - // So we always use new_from_previous which preserves base_paths - let mut prev_manifest = - Manifest::new_from_previous(current_manifest, schema, Arc::new(final_fragments)); - - if let (Some(user_requested_version), Operation::Overwrite { .. }) = - (user_requested_version, &self.operation) - { - // If this is an overwrite operation and the user has requested a specific version - // then overwrite with that version. Otherwise, if the user didn't request a specific - // version, then overwrite with whatever version we had before. - prev_manifest.data_storage_format = DataStorageFormat::new(user_requested_version); - } - - prev_manifest - } else { - let data_storage_format = - Self::data_storage_format_from_files(&final_fragments, user_requested_version)?; - Manifest::new( - schema, - Arc::new(final_fragments), - data_storage_format, - reference_paths, - ) - }; - - manifest.tag.clone_from(&self.tag); - - if config.auto_set_feature_flags { - // Internal operations (e.g. CreateIndex) build with the default config, - // which has use_stable_row_ids = false. Without inheriting from the previous - // manifest, apply_feature_flags would clear FLAG_STABLE_ROW_IDS. - let inherited = current_manifest - .map(|m| m.uses_stable_row_ids()) - .unwrap_or(false); - let use_stable_row_ids = config.use_stable_row_ids || inherited; - apply_feature_flags( - &mut manifest, - use_stable_row_ids, - config.disable_transaction_file, - )?; - } - // Carried from the manifest this one is derived from. `new_from_previous` - // zeroes both feature words, so `apply_feature_flags` cannot see the - // previous state and every ordinary commit would otherwise drop the bit. - if let Some(current_manifest) = current_manifest { - inherit_mem_wal_index_catchup(&mut manifest, current_manifest)?; - } + let mut manifest = self.assemble_manifest( + current_manifest, + schema, + final_fragments, + reference_paths, + overwrite_storage_format, + config, + )?; // Set after apply_feature_flags, which resets both flag words: activation // is the one place the bit is turned on, and it must survive that reset. @@ -1418,9 +1362,6 @@ impl Transaction { manifest.writer_feature_flags |= FLAG_MEM_WAL_INDEX_CATCHUP; } - manifest.set_timestamp(config.timestamp_nanos); - - manifest.update_max_fragment_id(); match &self.operation { Operation::Overwrite { @@ -1604,6 +1545,106 @@ impl Transaction { Ok((manifest, final_indices)) } + /// Put an assembled fragment list into the shape the manifest requires: + /// ordered by id, with no data file left holding only tombstoned fields, and + /// with each fragment's overlays in oldest-to-newest order. + pub(super) fn normalize_fragments(fragments: &mut Vec) -> Result<()> { + // If a fragment was reserved then it may not belong at the end of the list. + fragments.sort_by_key(|frag| frag.id); + + Self::remove_tombstoned_data_files(fragments); + + // Enforce the newest-last overlay ordering invariant at the write + // boundary. Load normalizes with a sort; this rejects any commit path + // that assembled a fragment's overlays out of order. + for fragment in fragments.iter() { + if !fragment.overlays.is_empty() { + crate::format::overlay::verify_overlays_newest_last(&fragment.overlays)?; + } + } + Ok(()) + } + + fn user_requested_version(config: &ManifestBuildConfig) -> Option { + match (&config.storage_format, config.use_legacy_format) { + (Some(storage_format), _) => Some(storage_format.lance_file_format()), + (None, Some(true)) => Some(ConcreteFileVersion::V1), + (None, Some(false)) => Some(ConcreteFileVersion::V2_0), + (None, None) => None, + } + } + + /// Build the manifest itself from an already-decided schema and fragment + /// list, and apply the settings that every operation shares: the tag, + /// feature flags, timestamp, and fragment id watermark. + /// + /// `overwrite_storage_format` replaces the inherited storage format with the + /// user-requested one, for operations that rewrite the whole dataset. + pub(super) fn assemble_manifest( + &self, + current_manifest: Option<&Manifest>, + schema: lance_core::datatypes::Schema, + fragments: Vec, + reference_paths: HashMap, + overwrite_storage_format: bool, + config: &ManifestBuildConfig, + ) -> Result { + let user_requested_version = Self::user_requested_version(config); + + let mut manifest = if let Some(current_manifest) = current_manifest { + // OVERWRITE with initial_bases on existing dataset is not allowed (caught by validation) + // So we always use new_from_previous which preserves base_paths + let mut prev_manifest = + Manifest::new_from_previous(current_manifest, schema, Arc::new(fragments)); + + if let (true, Some(user_requested_version)) = + (overwrite_storage_format, user_requested_version) + { + prev_manifest.data_storage_format = DataStorageFormat::new(user_requested_version); + } + + prev_manifest + } else { + let data_storage_format = + Self::data_storage_format_from_files(&fragments, user_requested_version)?; + Manifest::new( + schema, + Arc::new(fragments), + data_storage_format, + reference_paths, + ) + }; + + manifest.tag.clone_from(&self.tag); + + if config.auto_set_feature_flags { + // Internal operations (e.g. CreateIndex) build with the default config, + // which has use_stable_row_ids = false. Without inheriting from the previous + // manifest, apply_feature_flags would clear FLAG_STABLE_ROW_IDS. + let inherited = current_manifest + .map(|m| m.uses_stable_row_ids()) + .unwrap_or(false); + let use_stable_row_ids = config.use_stable_row_ids || inherited; + apply_feature_flags( + &mut manifest, + use_stable_row_ids, + config.disable_transaction_file, + )?; + } + // Carried from the manifest this one is derived from. `new_from_previous` + // zeroes both feature words, so `apply_feature_flags` cannot see the + // previous state and every ordinary commit would otherwise drop the bit. + if let Some(current_manifest) = current_manifest { + inherit_mem_wal_index_catchup(&mut manifest, current_manifest)?; + } + + manifest.set_timestamp(config.timestamp_nanos); + + manifest.update_max_fragment_id(); + + Ok(manifest) + } + /// Remove data files that only contain tombstoned fields (-2) /// These files no longer contain any live data and can be safely dropped fn remove_tombstoned_data_files(fragments: &mut [Fragment]) { From 564188736ecb424cac8c9fecc30ee2fadac5f492 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sat, 15 Aug 2026 17:12:42 -0700 Subject: [PATCH 04/24] feat(transaction): apply the minting actions Adds the action apply path: `build_manifest_from_actions` walks the action set in order against a working copy of the read-version state, and the four minting actions (AddFragment, AddDataFile, AddField, AddBase) allocate ids from the target's counters as they are reached. Each mint records its id against the action's local token, so a later action in the same operation resolves that token to the id this apply chose. The same action set replayed against a different version therefore lands on different ids without any action changing -- the property branch merge needs. An action set requires an existing dataset: it is a delta, so there is nothing for it to be a delta against at creation time. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance-table/src/transaction/action.rs | 2 + .../src/transaction/action/apply.rs | 592 ++++++++++++++++++ .../src/transaction/manifest_build.rs | 17 +- 3 files changed, 608 insertions(+), 3 deletions(-) create mode 100644 rust/lance-table/src/transaction/action/apply.rs diff --git a/rust/lance-table/src/transaction/action.rs b/rust/lance-table/src/transaction/action.rs index f831044c708..6923cb9d312 100644 --- a/rust/lance-table/src/transaction/action.rs +++ b/rust/lance-table/src/transaction/action.rs @@ -16,6 +16,7 @@ //! //! ```text //! action the vocabulary (this module) +//! action::apply applying an action set to produce the next manifest //! action::proto its persisted protobuf encoding //! ``` //! @@ -25,6 +26,7 @@ //! contract, and a transaction carrying a [`UserOperation`] is rejected outright //! by libraries that predate it. +mod apply; mod proto; use crate::format::{BasePath, DataFile, DeletionFile, RowIdMeta}; diff --git a/rust/lance-table/src/transaction/action/apply.rs b/rust/lance-table/src/transaction/action/apply.rs new file mode 100644 index 00000000000..089da02f282 --- /dev/null +++ b/rust/lance-table/src/transaction/action/apply.rs @@ -0,0 +1,592 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Applying an action set to produce the next manifest. +//! +//! Each action is applied in order against a working copy of the read-version +//! state. Minting actions allocate an id from the target's counters as they are +//! reached and record it against their local token, so every later reference to +//! that token -- in this same operation -- resolves to the id this apply chose. +//! Replaying the same action set against a different version therefore produces +//! different ids without any of the actions changing. + +use super::{Action, AddBase, AddDataFile, AddField, AddFragment, Ref, UserOperation}; +use crate::format::{BasePath, Fragment, IndexMetadata, Manifest, ManifestBuildConfig}; +use crate::rowids::version::build_version_meta; +use crate::transaction::Transaction; +use lance_core::datatypes::{Field, Schema}; +use lance_core::{Error, Result}; +use std::collections::{HashMap, HashSet}; + +impl Transaction { + /// Build the next manifest by applying an action set. + /// + /// Unlike the legacy path this always requires a current manifest: an action + /// set describes a delta, so there is nothing for it to be a delta against + /// when the dataset does not exist yet. + pub(in crate::transaction) fn build_manifest_from_actions( + &self, + user_operation: &UserOperation, + current_manifest: Option<&Manifest>, + current_indices: Vec, + transaction_file_path: &str, + config: &ManifestBuildConfig, + ) -> Result<(Manifest, Vec)> { + let current_manifest = current_manifest.ok_or_else(|| { + Error::invalid_input( + "an action-based transaction describes a change to an existing dataset; \ + it cannot create one", + ) + })?; + if config.use_stable_row_ids && !current_manifest.uses_stable_row_ids() { + return Err(Error::not_supported_source( + "Cannot enable stable row ids on existing dataset".into(), + )); + } + + let new_version = current_manifest.version + 1; + let mut state = ApplyState::new(current_manifest); + for action in user_operation.iter_actions() { + state.apply(action)?; + } + + let mut next_row_id = current_manifest + .uses_stable_row_ids() + .then_some(current_manifest.next_row_id); + state.assign_row_ids_to_minted_fragments(&mut next_row_id, new_version)?; + + let ApplyState { + schema, + mut fragments, + new_bases, + .. + } = state; + + let mut indices = current_indices; + Self::retain_relevant_indices(&mut indices, &schema, &fragments); + + Self::normalize_fragments(&mut fragments)?; + let mut manifest = self.assemble_manifest( + Some(current_manifest), + schema, + fragments, + HashMap::new(), + false, + config, + )?; + + for base in new_bases { + manifest.base_paths.insert(base.id, base); + } + + manifest.transaction_file = Some(transaction_file_path.to_string()); + if let Some(next_row_id) = next_row_id { + manifest.next_row_id = next_row_id; + } + + Ok((manifest, indices)) + } +} + +/// The read-version state an action set is applied against, plus the id +/// allocations made so far. +struct ApplyState { + schema: Schema, + fragments: Vec, + /// Base paths minted by this operation. Kept apart from the manifest's own + /// base paths, which the manifest assembly inherits from the read version. + new_bases: Vec, + existing_base_paths: HashMap, + + next_fragment_id: u64, + next_field_id: i32, + next_base_id: u32, + + /// Local token -> the id minted for it, one map per id space. + fragment_tokens: HashMap, + field_tokens: HashMap, + base_tokens: HashMap, + + /// Ids of the fragments this operation minted. + minted_fragments: HashSet, +} + +impl ApplyState { + fn new(manifest: &Manifest) -> Self { + Self { + schema: manifest.schema.clone(), + fragments: manifest.fragments.as_ref().clone(), + new_bases: Vec::new(), + existing_base_paths: manifest.base_paths.clone(), + next_fragment_id: manifest.max_fragment_id().map(|id| id + 1).unwrap_or(0), + next_field_id: manifest.max_field_id() + 1, + next_base_id: manifest + .base_paths + .keys() + .max() + .map(|id| id + 1) + .unwrap_or(1), + fragment_tokens: HashMap::new(), + field_tokens: HashMap::new(), + base_tokens: HashMap::new(), + minted_fragments: HashSet::new(), + } + } + + fn apply(&mut self, action: &Action) -> Result<()> { + match action { + Action::AddFragment(action) => self.add_fragment(action), + Action::AddDataFile(action) => self.add_data_file(action), + Action::AddField(action) => self.add_field(action), + Action::AddBase(action) => self.add_base(action), + other => Err(Error::not_supported(format!( + "applying the {other} action is not implemented yet" + ))), + } + } + + fn add_fragment(&mut self, action: &AddFragment) -> Result<()> { + if self.fragment_tokens.contains_key(&action.local) { + return Err(duplicate_token_err("fragment", action.local)); + } + let id = self.next_fragment_id; + self.next_fragment_id += 1; + self.fragment_tokens.insert(action.local, id); + self.minted_fragments.insert(id); + + self.fragments.push(Fragment { + id, + files: Vec::new(), + overlays: Vec::new(), + deletion_file: None, + row_id_meta: action.row_id_meta.clone(), + physical_rows: Some(action.physical_rows as usize), + last_updated_at_version_meta: action.last_updated_at_version_meta.clone(), + created_at_version_meta: action.created_at_version_meta.clone(), + }); + Ok(()) + } + + fn add_data_file(&mut self, action: &AddDataFile) -> Result<()> { + let fragment_id = self.resolve_fragment(action.fragment)?; + let field_ids = action + .field_ids + .iter() + .map(|field| self.resolve_field(*field)) + .collect::>>()?; + + let mut file = action.file.clone(); + if !file.column_indices.is_empty() && file.column_indices.len() != field_ids.len() { + return Err(Error::invalid_input(format!( + "AddDataFile for fragment {fragment_id} lists {} field ids but the file has {} \ + columns", + field_ids.len(), + file.column_indices.len() + ))); + } + file.fields = field_ids.into(); + + let fragment = self + .fragments + .iter_mut() + .find(|fragment| fragment.id == fragment_id) + .ok_or_else(|| { + Error::invalid_input(format!( + "AddDataFile targets fragment {fragment_id}, which does not exist" + )) + })?; + fragment.files.push(file); + Ok(()) + } + + fn add_field(&mut self, action: &AddField) -> Result<()> { + let id = self.next_field_id; + self.next_field_id += 1; + if self.field_tokens.contains_key(&action.local) { + return Err(duplicate_token_err("field", action.local)); + } + self.field_tokens.insert(action.local, id); + + let parent_id = action + .parent + .map(|parent| self.resolve_field(parent)) + .transpose()?; + + // The definition's own id, parent id, and children are ignored: the + // minted id and the parent reference carry that structure, and each + // child column arrives as its own action. + let field = Field { + id, + parent_id: parent_id.unwrap_or(-1), + children: Vec::new(), + ..action.def.clone() + }; + + match parent_id { + None => self.schema.fields.push(field), + Some(parent_id) => { + let parent = self.schema.field_by_id_mut(parent_id).ok_or_else(|| { + Error::invalid_input(format!( + "AddField names parent field {parent_id}, which does not exist" + )) + })?; + parent.children.push(field); + } + } + Ok(()) + } + + fn add_base(&mut self, action: &AddBase) -> Result<()> { + let id = self.next_base_id; + self.next_base_id += 1; + if self.base_tokens.contains_key(&action.local) { + return Err(duplicate_token_err("base", action.local)); + } + self.base_tokens.insert(action.local, id); + + let conflicting = self + .existing_base_paths + .values() + .chain(self.new_bases.iter()) + .find(|base| base.name == action.base.name || base.path == action.base.path); + if let Some(conflicting) = conflicting { + return Err(Error::invalid_input(format!( + "Conflict detected: Base path with name '{:?}' or path '{}' already exists. \ + Existing: name='{:?}', path='{}'", + action.base.name, action.base.path, conflicting.name, conflicting.path + ))); + } + + let mut base = action.base.clone(); + base.id = id; + self.new_bases.push(base); + Ok(()) + } + + /// Stamp row ids and version metadata onto the fragments this operation + /// minted, matching what an Append does for its new fragments. + fn assign_row_ids_to_minted_fragments( + &mut self, + next_row_id: &mut Option, + new_version: u64, + ) -> Result<()> { + let Some(next_row_id) = next_row_id.as_mut() else { + return Ok(()); + }; + let minted_ids = std::mem::take(&mut self.minted_fragments); + // The manifest assembly sorts fragments by id, so partitioning them here + // does not disturb the final order. + let (mut minted, existing): (Vec, Vec) = self + .fragments + .drain(..) + .partition(|fragment| minted_ids.contains(&fragment.id)); + + Transaction::assign_row_ids(next_row_id, minted.as_mut_slice())?; + for fragment in minted.iter_mut() { + // An action may carry its own sequences (a squashed operation + // does); only stamp the ones it left for apply to fill. + let version_meta = build_version_meta(fragment, new_version); + if fragment.last_updated_at_version_meta.is_none() { + fragment.last_updated_at_version_meta = version_meta.clone(); + } + if fragment.created_at_version_meta.is_none() { + fragment.created_at_version_meta = version_meta; + } + } + + self.fragments = existing; + self.fragments.extend(minted); + self.minted_fragments = minted_ids; + Ok(()) + } + + fn resolve_fragment(&self, reference: Ref) -> Result { + match reference { + Ref::Committed(id) => Ok(id), + Ref::Local(token) => self + .fragment_tokens + .get(&token) + .copied() + .ok_or_else(|| unbound_token_err("fragment", token)), + } + } + + fn resolve_field(&self, reference: Ref) -> Result { + match reference { + Ref::Committed(id) => i32::try_from(id).map_err(|_| { + Error::invalid_input(format!("field id {id} in an action is out of range")) + }), + Ref::Local(token) => self + .field_tokens + .get(&token) + .copied() + .ok_or_else(|| unbound_token_err("field", token)), + } + } +} + +fn unbound_token_err(space: &str, token: u32) -> Error { + Error::invalid_input(format!( + "an action references local {space} token {token}, which no earlier action in this \ + operation minted" + )) +} + +fn duplicate_token_err(space: &str, token: u32) -> Error { + Error::invalid_input(format!( + "local {space} token {token} is minted more than once in this operation; tokens must be \ + distinct within an operation" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::DataFile; + use crate::transaction::Operation; + use crate::transaction::action::UserAction; + use crate::transaction::test_support::{default_build_config, sample_manifest}; + use arrow_schema::{DataType, Field as ArrowField}; + + fn apply(manifest: &Manifest, actions: Vec) -> Result { + apply_with_indices(manifest, actions).map(|(manifest, _)| manifest) + } + + fn apply_with_indices( + manifest: &Manifest, + actions: Vec, + ) -> Result<(Manifest, Vec)> { + let transaction = Transaction::new( + manifest.version, + Operation::UserOperation(UserOperation::new( + "test", + vec![UserAction::new("step", actions)], + )), + None, + ); + transaction.build_manifest( + Some(manifest), + Vec::new(), + "tx.txn", + &default_build_config(), + ) + } + + fn added_field(name: &str) -> Field { + Field::try_from(ArrowField::new(name, DataType::Int32, true)).unwrap() + } + + #[test] + fn test_add_fragment_and_data_file_mint_ids() { + let manifest = sample_manifest(); + let next = apply( + &manifest, + vec![ + Action::AddFragment(AddFragment { + local: 0, + physical_rows: 10, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Local(0), + file: DataFile::new_unstarted("data/new.lance", 2, 0), + field_ids: vec![Ref::Committed(0)], + data_change: true, + }), + ], + ) + .unwrap(); + + // sample_manifest already holds fragment 0, so the mint lands on 1. + assert_eq!( + next.fragments.iter().map(|f| f.id).collect::>(), + vec![0, 1] + ); + let minted = next.fragments.iter().find(|f| f.id == 1).unwrap(); + assert_eq!(minted.physical_rows, Some(10)); + assert_eq!(minted.files.len(), 1); + // The file's field list is stamped in from the action's refs. + assert_eq!(minted.files[0].fields.as_ref(), &[0]); + assert_eq!(next.max_fragment_id(), Some(1)); + } + + #[test] + fn test_two_add_fields_mint_distinct_ids() { + let manifest = sample_manifest(); + let next = apply( + &manifest, + vec![ + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("a"), + }), + Action::AddField(AddField { + local: 1, + parent: None, + def: added_field("b"), + }), + ], + ) + .unwrap(); + + let ids = next + .schema + .fields + .iter() + .map(|f| (f.name.as_str(), f.id)) + .collect::>(); + assert_eq!(ids, vec![("id", 0), ("a", 1), ("b", 2)]); + } + + #[test] + fn test_add_field_then_add_its_data_file() { + // The add-column shape: mint the field, then write the file that backs + // it, naming the field by the token the mint has not resolved yet. + let manifest = sample_manifest(); + let next = apply( + &manifest, + vec![ + Action::AddField(AddField { + local: 7, + parent: None, + def: added_field("added"), + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Committed(0), + file: DataFile::new_unstarted("data/added.lance", 2, 0), + field_ids: vec![Ref::Local(7)], + data_change: true, + }), + ], + ) + .unwrap(); + + let field_id = next.schema.field("added").unwrap().id; + assert_eq!(field_id, 1); + let fragment = next.fragments.iter().find(|f| f.id == 0).unwrap(); + assert_eq!(fragment.files.last().unwrap().fields.as_ref(), &[field_id]); + } + + #[test] + fn test_add_field_under_a_parent() { + let manifest = sample_manifest(); + let next = apply( + &manifest, + vec![ + Action::AddField(AddField { + local: 0, + parent: None, + def: Field::try_from(ArrowField::new( + "nested", + DataType::Struct(Default::default()), + true, + )) + .unwrap(), + }), + Action::AddField(AddField { + local: 1, + parent: Some(Ref::Local(0)), + def: added_field("child"), + }), + ], + ) + .unwrap(); + + let parent = next.schema.field("nested").unwrap(); + assert_eq!(parent.children.len(), 1); + assert_eq!(parent.children[0].name, "child"); + assert_eq!(parent.children[0].parent_id, parent.id); + } + + #[test] + fn test_add_base_mints_an_id_and_rejects_duplicates() { + let manifest = sample_manifest(); + let next = apply( + &manifest, + vec![Action::AddBase(AddBase { + local: 0, + base: BasePath::new(0, "s3://bucket/a".into(), Some("a".into()), false), + })], + ) + .unwrap(); + assert_eq!(next.base_paths.len(), 1); + assert_eq!(next.base_paths[&1].path, "s3://bucket/a"); + + let error = apply( + &next, + vec![Action::AddBase(AddBase { + local: 0, + base: BasePath::new(0, "s3://bucket/a".into(), Some("other".into()), false), + })], + ) + .unwrap_err(); + assert!( + error.to_string().contains("already exists"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_unbound_local_token_is_rejected() { + let manifest = sample_manifest(); + let error = apply( + &manifest, + vec![Action::AddDataFile(AddDataFile { + fragment: Ref::Local(3), + file: DataFile::new_unstarted("data/x.lance", 2, 0), + field_ids: vec![Ref::Committed(0)], + data_change: true, + })], + ) + .unwrap_err(); + assert!( + error.to_string().contains("local fragment token 3"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_duplicate_local_token_is_rejected() { + let manifest = sample_manifest(); + let error = apply( + &manifest, + vec![ + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("a"), + }), + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("b"), + }), + ], + ) + .unwrap_err(); + assert!( + error.to_string().contains("minted more than once"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_action_set_cannot_create_a_dataset() { + let transaction = Transaction::new( + 0, + Operation::UserOperation(UserOperation::new("test", vec![])), + None, + ); + let error = transaction + .build_manifest(None, Vec::new(), "tx.txn", &default_build_config()) + .unwrap_err(); + assert!( + error.to_string().contains("cannot create one"), + "unexpected error: {error}" + ); + } +} diff --git a/rust/lance-table/src/transaction/manifest_build.rs b/rust/lance-table/src/transaction/manifest_build.rs index acc61ab5090..cac9bcd17c2 100644 --- a/rust/lance-table/src/transaction/manifest_build.rs +++ b/rust/lance-table/src/transaction/manifest_build.rs @@ -463,6 +463,16 @@ impl Transaction { config: &ManifestBuildConfig, read_version_state: Option>, ) -> Result<(Manifest, Vec)> { + if let Operation::UserOperation(user_operation) = &self.operation { + return self.build_manifest_from_actions( + user_operation, + current_manifest, + current_indices, + transaction_file_path, + config, + ); + } + if config.use_stable_row_ids && config.migration_next_row_id.is_none() && current_manifest @@ -1288,8 +1298,9 @@ impl Transaction { final_fragments.extend(maybe_existing_fragments?.clone()); } Operation::UserOperation(_) => { - return Err(Error::not_supported( - "applying an action-based transaction is not implemented yet", + // Handled by build_manifest_from_actions before this match. + return Err(Error::internal( + "an action-based operation reached the legacy manifest build".to_string(), )); } }; @@ -1548,7 +1559,7 @@ impl Transaction { /// Put an assembled fragment list into the shape the manifest requires: /// ordered by id, with no data file left holding only tombstoned fields, and /// with each fragment's overlays in oldest-to-newest order. - pub(super) fn normalize_fragments(fragments: &mut Vec) -> Result<()> { + pub(super) fn normalize_fragments(fragments: &mut [Fragment]) -> Result<()> { // If a fragment was reserved then it may not belong at the end of the list. fragments.sort_by_key(|frag| frag.id); From a44c8847e42dc8323cdd6382fc3cb6d76452a4d7 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sat, 15 Aug 2026 17:20:10 -0700 Subject: [PATCH 05/24] feat(transaction): apply the reference-stable actions Applies TombstoneFieldData, RemoveFragment, SetDeletionFile, and AlterField, completing the eight-action apply path. Tombstoning a field's data and retyping a field both leave any index over that field describing values the fragment no longer holds, so the affected fragments are dropped from the index bitmap rather than the index being discarded outright. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/transaction/action/apply.rs | 433 +++++++++++++++++- 1 file changed, 420 insertions(+), 13 deletions(-) diff --git a/rust/lance-table/src/transaction/action/apply.rs b/rust/lance-table/src/transaction/action/apply.rs index 089da02f282..dd347668314 100644 --- a/rust/lance-table/src/transaction/action/apply.rs +++ b/rust/lance-table/src/transaction/action/apply.rs @@ -10,7 +10,10 @@ //! Replaying the same action set against a different version therefore produces //! different ids without any of the actions changing. -use super::{Action, AddBase, AddDataFile, AddField, AddFragment, Ref, UserOperation}; +use super::{ + Action, AddBase, AddDataFile, AddField, AddFragment, AlterField, Ref, RemoveFragment, + SetDeletionFile, TombstoneFieldData, UserOperation, +}; use crate::format::{BasePath, Fragment, IndexMetadata, Manifest, ManifestBuildConfig}; use crate::rowids::version::build_version_meta; use crate::transaction::Transaction; @@ -18,6 +21,10 @@ use lance_core::datatypes::{Field, Schema}; use lance_core::{Error, Result}; use std::collections::{HashMap, HashSet}; +/// The field id written into a data file's field list once the file no longer +/// backs that field. A file whose every slot is tombstoned is dropped. +const TOMBSTONED_FIELD: i32 = -2; + impl Transaction { /// Build the next manifest by applying an action set. /// @@ -59,10 +66,12 @@ impl Transaction { schema, mut fragments, new_bases, + rebound_fields, .. } = state; let mut indices = current_indices; + prune_rebound_fields_from_indices(&mut indices, &rebound_fields); Self::retain_relevant_indices(&mut indices, &schema, &fragments); Self::normalize_fragments(&mut fragments)?; @@ -109,6 +118,10 @@ struct ApplyState { /// Ids of the fragments this operation minted. minted_fragments: HashSet, + + /// Fields whose backing data changed, per fragment. An index covering such + /// a field no longer describes that fragment's contents. + rebound_fields: HashMap>, } impl ApplyState { @@ -130,6 +143,7 @@ impl ApplyState { field_tokens: HashMap::new(), base_tokens: HashMap::new(), minted_fragments: HashSet::new(), + rebound_fields: HashMap::new(), } } @@ -139,9 +153,10 @@ impl ApplyState { Action::AddDataFile(action) => self.add_data_file(action), Action::AddField(action) => self.add_field(action), Action::AddBase(action) => self.add_base(action), - other => Err(Error::not_supported(format!( - "applying the {other} action is not implemented yet" - ))), + Action::TombstoneFieldData(action) => self.tombstone_field_data(action), + Action::RemoveFragment(action) => self.remove_fragment(action), + Action::SetDeletionFile(action) => self.set_deletion_file(action), + Action::AlterField(action) => self.alter_field(action), } } @@ -263,6 +278,93 @@ impl ApplyState { Ok(()) } + fn tombstone_field_data(&mut self, action: &TombstoneFieldData) -> Result<()> { + let fragment_id = self.resolve_fragment(action.fragment)?; + let fragment = fragment_mut(&mut self.fragments, fragment_id, "TombstoneFieldData")?; + + for &field_id in &action.field_ids { + let mut found = false; + for file in fragment.files.iter_mut() { + let Some(position) = file.fields.iter().position(|id| *id == field_id) else { + continue; + }; + let mut fields = file.fields.to_vec(); + fields[position] = TOMBSTONED_FIELD; + file.fields = fields.into(); + found = true; + } + if !found { + return Err(Error::invalid_input(format!( + "TombstoneFieldData names field {field_id}, which no data file in fragment \ + {fragment_id} backs" + ))); + } + } + + // New values for these fields supersede any overlay still shadowing + // them, so the drop is not silently masked by stale overlay cells. + let overlaid: Vec = action + .field_ids + .iter() + .filter_map(|id| u32::try_from(*id).ok()) + .collect(); + crate::format::overlay::tombstone_overlay_fields(&mut fragment.overlays, &overlaid); + + self.rebound_fields + .entry(fragment_id) + .or_default() + .extend(action.field_ids.iter().copied()); + Ok(()) + } + + fn remove_fragment(&mut self, action: &RemoveFragment) -> Result<()> { + let fragment_id = self.resolve_fragment(action.fragment)?; + let before = self.fragments.len(); + self.fragments.retain(|fragment| fragment.id != fragment_id); + if self.fragments.len() == before { + return Err(Error::invalid_input(format!( + "RemoveFragment targets fragment {fragment_id}, which does not exist" + ))); + } + self.minted_fragments.remove(&fragment_id); + self.rebound_fields.remove(&fragment_id); + Ok(()) + } + + fn set_deletion_file(&mut self, action: &SetDeletionFile) -> Result<()> { + let fragment = fragment_mut(&mut self.fragments, action.fragment, "SetDeletionFile")?; + fragment.deletion_file = action.deletion_file.clone(); + Ok(()) + } + + fn alter_field(&mut self, action: &AlterField) -> Result<()> { + let field = self.schema.field_by_id_mut(action.field).ok_or_else(|| { + Error::invalid_input(format!( + "AlterField names field {}, which does not exist", + action.field + )) + })?; + if let Some(name) = &action.name { + field.name.clone_from(name); + } + if let Some(nullable) = action.nullable { + field.nullable = nullable; + } + if let Some(logical_type) = &action.logical_type { + field.logical_type = logical_type.as_str().into(); + // The cast leaves any index on the field describing the old type. + // The data rewrite itself is separate actions; this only records + // that every fragment's view of the field changed. + for fragment in &self.fragments { + self.rebound_fields + .entry(fragment.id) + .or_default() + .insert(action.field); + } + } + Ok(()) + } + /// Stamp row ids and version metadata onto the fragments this operation /// minted, matching what an Append does for its new fragments. fn assign_row_ids_to_minted_fragments( @@ -325,6 +427,48 @@ impl ApplyState { } } +fn fragment_mut<'a>( + fragments: &'a mut [Fragment], + fragment_id: u64, + action: &str, +) -> Result<&'a mut Fragment> { + fragments + .iter_mut() + .find(|fragment| fragment.id == fragment_id) + .ok_or_else(|| { + Error::invalid_input(format!( + "{action} targets fragment {fragment_id}, which does not exist" + )) + }) +} + +/// Drop the fragments whose data no longer matches what an index recorded. +/// +/// An index built over a field describes the values that were in that field +/// when it was built. Rebinding the field's data in a fragment invalidates the +/// index for that fragment only, so the fragment leaves the bitmap rather than +/// the whole index being discarded. +fn prune_rebound_fields_from_indices( + indices: &mut [IndexMetadata], + rebound: &HashMap>, +) { + if rebound.is_empty() { + return; + } + for index in indices.iter_mut() { + let Some(bitmap) = index.fragment_bitmap.as_mut() else { + continue; + }; + for (fragment_id, fields) in rebound { + if index.fields.iter().any(|field| fields.contains(field)) + && let Ok(fragment_id) = u32::try_from(*fragment_id) + { + bitmap.remove(fragment_id); + } + } + } +} + fn unbound_token_err(space: &str, token: u32) -> Error { Error::invalid_input(format!( "an action references local {space} token {token}, which no earlier action in this \ @@ -342,19 +486,23 @@ fn duplicate_token_err(space: &str, token: u32) -> Error { #[cfg(test)] mod tests { use super::*; - use crate::format::DataFile; + use crate::format::{DataFile, DeletionFile, DeletionFileType}; use crate::transaction::Operation; use crate::transaction::action::UserAction; - use crate::transaction::test_support::{default_build_config, sample_manifest}; + use crate::transaction::test_support::{ + default_build_config, sample_index_metadata, sample_manifest, + }; use arrow_schema::{DataType, Field as ArrowField}; + use std::sync::Arc; fn apply(manifest: &Manifest, actions: Vec) -> Result { - apply_with_indices(manifest, actions).map(|(manifest, _)| manifest) + apply_with_indices(manifest, actions, Vec::new()).map(|(manifest, _)| manifest) } fn apply_with_indices( manifest: &Manifest, actions: Vec, + indices: Vec, ) -> Result<(Manifest, Vec)> { let transaction = Transaction::new( manifest.version, @@ -364,18 +512,277 @@ mod tests { )), None, ); - transaction.build_manifest( - Some(manifest), - Vec::new(), - "tx.txn", - &default_build_config(), - ) + transaction.build_manifest(Some(manifest), indices, "tx.txn", &default_build_config()) } fn added_field(name: &str) -> Field { Field::try_from(ArrowField::new(name, DataType::Int32, true)).unwrap() } + /// `sample_manifest` with fragment 0 actually backed by a data file, so the + /// reference-stable actions have something committed to point at. + fn backed_manifest() -> Manifest { + let mut manifest = sample_manifest(); + let mut fragment = Fragment::new(0); + fragment.physical_rows = Some(10); + fragment.files.push(DataFile::new( + "data/0.lance", + vec![0], + vec![0], + 2, + 0, + None, + None, + )); + manifest.fragments = Arc::new(vec![fragment]); + manifest + } + + #[test] + fn test_tombstone_field_data_drops_the_backing_file() { + let manifest = backed_manifest(); + let next = apply( + &manifest, + vec![Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Committed(0), + field_ids: vec![0], + data_change: true, + })], + ) + .unwrap(); + + // The file backed only field 0, so tombstoning it leaves nothing behind. + assert!(next.fragments[0].files.is_empty()); + } + + #[test] + fn test_tombstone_field_data_keeps_a_file_with_a_live_field() { + let mut manifest = backed_manifest(); + let mut fragment = manifest.fragments[0].clone(); + fragment.files[0] = DataFile::new("data/0.lance", vec![0, 1], vec![0, 1], 2, 0, None, None); + manifest.fragments = Arc::new(vec![fragment]); + + let next = apply( + &manifest, + vec![Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Committed(0), + field_ids: vec![0], + data_change: true, + })], + ) + .unwrap(); + + assert_eq!( + next.fragments[0].files[0].fields.as_ref(), + &[TOMBSTONED_FIELD, 1] + ); + } + + #[test] + fn test_tombstone_field_data_prunes_the_fragment_from_covering_indices() { + let manifest = backed_manifest(); + let (_, indices) = apply_with_indices( + &manifest, + vec![Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Committed(0), + field_ids: vec![0], + data_change: true, + })], + vec![sample_index_metadata("idx")], + ) + .unwrap(); + + // The index covers field 0, whose data in fragment 0 is now gone. + assert!(indices[0].fragment_bitmap.as_ref().unwrap().is_empty()); + } + + #[test] + fn test_tombstone_field_data_rejects_a_field_no_file_backs() { + let manifest = backed_manifest(); + let error = apply( + &manifest, + vec![Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Committed(0), + field_ids: vec![7], + data_change: true, + })], + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + assert!(error.to_string().contains("field 7"), "{error}"); + } + + #[test] + fn test_remove_fragment() { + let manifest = backed_manifest(); + let next = apply( + &manifest, + vec![Action::RemoveFragment(RemoveFragment { + fragment: Ref::Committed(0), + data_change: true, + })], + ) + .unwrap(); + + assert!(next.fragments.is_empty()); + } + + #[test] + fn test_remove_fragment_can_drop_one_minted_in_the_same_operation() { + let manifest = backed_manifest(); + let next = apply( + &manifest, + vec![ + Action::AddFragment(AddFragment { + local: 0, + physical_rows: 10, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }), + Action::RemoveFragment(RemoveFragment { + fragment: Ref::Local(0), + data_change: true, + }), + ], + ) + .unwrap(); + + assert_eq!( + next.fragments.iter().map(|f| f.id).collect::>(), + vec![0] + ); + } + + #[test] + fn test_remove_fragment_rejects_a_missing_fragment() { + let manifest = backed_manifest(); + let error = apply( + &manifest, + vec![Action::RemoveFragment(RemoveFragment { + fragment: Ref::Committed(7), + data_change: true, + })], + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + assert!(error.to_string().contains("fragment 7"), "{error}"); + } + + #[test] + fn test_set_deletion_file_sets_and_clears() { + let manifest = backed_manifest(); + let deletion_file = DeletionFile { + read_version: manifest.version, + id: 3, + file_type: DeletionFileType::Array, + num_deleted_rows: Some(2), + base_id: None, + }; + let next = apply( + &manifest, + vec![Action::SetDeletionFile(SetDeletionFile { + fragment: 0, + deletion_file: Some(deletion_file.clone()), + data_change: true, + })], + ) + .unwrap(); + assert_eq!(next.fragments[0].deletion_file, Some(deletion_file)); + + // An absent deletion file is a request to clear it, not a no-op. + let cleared = apply( + &next, + vec![Action::SetDeletionFile(SetDeletionFile { + fragment: 0, + deletion_file: None, + data_change: true, + })], + ) + .unwrap(); + assert_eq!(cleared.fragments[0].deletion_file, None); + } + + #[test] + fn test_set_deletion_file_rejects_a_missing_fragment() { + let manifest = backed_manifest(); + let error = apply( + &manifest, + vec![Action::SetDeletionFile(SetDeletionFile { + fragment: 7, + deletion_file: None, + data_change: true, + })], + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + } + + #[test] + fn test_alter_field_renames_without_touching_indices() { + let manifest = backed_manifest(); + let (next, indices) = apply_with_indices( + &manifest, + vec![Action::AlterField(AlterField { + field: 0, + name: Some("renamed".into()), + logical_type: None, + nullable: Some(true), + })], + vec![sample_index_metadata("idx")], + ) + .unwrap(); + + let field = next.schema.field_by_id(0).unwrap(); + assert_eq!(field.name, "renamed"); + assert!(field.nullable); + // A rename does not change the values the index recorded. + assert!(indices[0].fragment_bitmap.as_ref().unwrap().contains(0)); + } + + #[test] + fn test_alter_field_retype_prunes_covering_indices() { + let manifest = backed_manifest(); + let (next, indices) = apply_with_indices( + &manifest, + vec![Action::AlterField(AlterField { + field: 0, + name: None, + logical_type: Some("int64".into()), + nullable: None, + })], + vec![sample_index_metadata("idx")], + ) + .unwrap(); + + assert_eq!( + next.schema.field_by_id(0).unwrap().logical_type.to_string(), + "int64" + ); + assert!(indices[0].fragment_bitmap.as_ref().unwrap().is_empty()); + } + + #[test] + fn test_alter_field_rejects_a_missing_field() { + let manifest = backed_manifest(); + let error = apply( + &manifest, + vec![Action::AlterField(AlterField { + field: 7, + name: Some("nope".into()), + ..Default::default() + })], + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + assert!(error.to_string().contains("field 7"), "{error}"); + } + #[test] fn test_add_fragment_and_data_file_mint_ids() { let manifest = sample_manifest(); From 3313acd93d81563f58b9ba9d3300af48bba139c8 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sat, 15 Aug 2026 17:23:52 -0700 Subject: [PATCH 06/24] feat(transaction): translate Append, Delete, and UpdateBases into actions Adds `TryFrom<&Operation> for Vec`, the recipe that turns a named operation into the granular actions it decomposes into. Squashing several operations into one commit is concatenation once both sides speak actions. Translation is fail-closed: an operation with no recipe yet, or one carrying a detail the actions cannot express, is rejected. Each case is covered by a parity test that builds the manifest twice -- once down the legacy path, once through the translation -- and asserts the two agree. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance-table/src/transaction/action.rs | 1 + .../src/transaction/action/translate.rs | 299 ++++++++++++++++++ 2 files changed, 300 insertions(+) create mode 100644 rust/lance-table/src/transaction/action/translate.rs diff --git a/rust/lance-table/src/transaction/action.rs b/rust/lance-table/src/transaction/action.rs index 6923cb9d312..62b1289906a 100644 --- a/rust/lance-table/src/transaction/action.rs +++ b/rust/lance-table/src/transaction/action.rs @@ -28,6 +28,7 @@ mod apply; mod proto; +mod translate; use crate::format::{BasePath, DataFile, DeletionFile, RowIdMeta}; use crate::rowids::version::RowDatasetVersionMeta; diff --git a/rust/lance-table/src/transaction/action/translate.rs b/rust/lance-table/src/transaction/action/translate.rs new file mode 100644 index 00000000000..a3a333b64e0 --- /dev/null +++ b/rust/lance-table/src/transaction/action/translate.rs @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Translating a legacy [`Operation`] into the action vocabulary. +//! +//! Each named operation is a fixed recipe over the granular actions. Expressing +//! the recipe here rather than in the commit loop is what lets several +//! operations be squashed into one commit: once both are action sets, combining +//! them is concatenation. +//! +//! Translation is fail-closed. An operation whose recipe is not written yet, or +//! one carrying a detail the actions cannot express, is rejected rather than +//! silently translated into something narrower. + +use super::UserAction; +use super::{Action, AddBase, AddDataFile, AddFragment, Ref, RemoveFragment, SetDeletionFile}; +use crate::format::Fragment; +use crate::transaction::Operation; +use lance_core::{Error, Result}; + +impl TryFrom<&Operation> for Vec { + type Error = Error; + + fn try_from(operation: &Operation) -> Result { + match operation { + Operation::Append { fragments } => Ok(vec![UserAction::new( + format!("append {} fragments", fragments.len()), + append_actions(fragments)?, + )]), + Operation::Delete { + updated_fragments, + deleted_fragment_ids, + predicate, + } => Ok(vec![UserAction::new( + format!("delete rows matching {predicate}"), + delete_actions(updated_fragments, deleted_fragment_ids), + )]), + Operation::UpdateBases { new_bases } => Ok(vec![UserAction::new( + format!("add {} base paths", new_bases.len()), + new_bases + .iter() + .enumerate() + .map(|(index, base)| { + Action::AddBase(AddBase { + local: index as u32, + base: base.clone(), + }) + }) + .collect(), + )]), + other => Err(Error::not_supported(format!( + "translating a {} operation into actions", + other.name() + ))), + } + } +} + +fn append_actions(fragments: &[Fragment]) -> Result> { + let mut actions = Vec::with_capacity(fragments.len() * 2); + for (index, fragment) in fragments.iter().enumerate() { + let local = index as u32; + let physical_rows = fragment.physical_rows.ok_or_else(|| { + Error::invalid_input( + "an appended fragment must know its physical row count to become an AddFragment", + ) + })?; + // A fragment being appended has no committed state, so anything that + // only makes sense against committed data means the caller built the + // operation by hand out of an existing fragment. + if fragment.deletion_file.is_some() || !fragment.overlays.is_empty() { + return Err(Error::invalid_input(format!( + "appended fragment {} carries a deletion file or overlays, which an append \ + cannot produce", + fragment.id + ))); + } + + actions.push(Action::AddFragment(AddFragment { + local, + physical_rows: physical_rows as u64, + row_id_meta: fragment.row_id_meta.clone(), + last_updated_at_version_meta: fragment.last_updated_at_version_meta.clone(), + created_at_version_meta: fragment.created_at_version_meta.clone(), + data_change: true, + })); + + for file in &fragment.files { + actions.push(Action::AddDataFile(AddDataFile { + fragment: Ref::Local(local), + file: file.clone(), + field_ids: committed_field_refs(file.fields.as_ref())?, + data_change: true, + })); + } + } + Ok(actions) +} + +/// A legacy delete replaces whole fragments, but the only thing it ever changes +/// on one is its deletion file, so that is what the translation carries over. +fn delete_actions(updated_fragments: &[Fragment], deleted_fragment_ids: &[u64]) -> Vec { + let mut actions = Vec::with_capacity(updated_fragments.len() + deleted_fragment_ids.len()); + for fragment in updated_fragments { + actions.push(Action::SetDeletionFile(SetDeletionFile { + fragment: fragment.id, + deletion_file: fragment.deletion_file.clone(), + data_change: true, + })); + } + for id in deleted_fragment_ids { + actions.push(Action::RemoveFragment(RemoveFragment { + fragment: Ref::Committed(*id), + data_change: true, + })); + } + actions +} + +fn committed_field_refs(field_ids: &[i32]) -> Result> { + field_ids + .iter() + .map(|id| { + u64::try_from(*id).map(Ref::Committed).map_err(|_| { + Error::invalid_input(format!( + "a data file in this operation lists field id {id}, which is not a committed \ + field" + )) + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::{ + BasePath, DataFile, DeletionFile, DeletionFileType, IndexMetadata, Manifest, RowIdMeta, + }; + use crate::rowids::{RowIdSequence, write_row_ids}; + use crate::transaction::Transaction; + use crate::transaction::action::UserOperation; + use crate::transaction::test_support::{ + default_build_config, make_stable_row_id_manifest, sample_manifest, + }; + use std::sync::Arc; + + /// Build the same manifest twice -- once down the legacy path, once by + /// translating the operation to actions -- and assert they agree. + fn assert_parity(manifest: &Manifest, operation: Operation) -> Manifest { + let (legacy, legacy_indices) = build(manifest, operation.clone()); + + let actions = Vec::::try_from(&operation).unwrap(); + let (translated, translated_indices) = build( + manifest, + Operation::UserOperation(UserOperation::new("translated", actions)), + ); + + assert_eq!(translated.fragments, legacy.fragments); + assert_eq!(translated.schema, legacy.schema); + assert_eq!(translated.base_paths, legacy.base_paths); + assert_eq!(translated.next_row_id, legacy.next_row_id); + assert_eq!(translated.max_fragment_id, legacy.max_fragment_id); + assert_eq!(translated_indices, legacy_indices); + translated + } + + fn build(manifest: &Manifest, operation: Operation) -> (Manifest, Vec) { + Transaction::new(manifest.version, operation, None) + .build_manifest( + Some(manifest), + Vec::new(), + "tx.txn", + &default_build_config(), + ) + .unwrap() + } + + fn appendable_fragment(path: &str) -> Fragment { + let mut fragment = Fragment::new(0); + fragment.physical_rows = Some(10); + fragment + .files + .push(DataFile::new(path, vec![0], vec![0], 2, 0, None, None)); + fragment + } + + fn manifest_with_fragments(fragments: Vec) -> Manifest { + let mut manifest = sample_manifest(); + manifest.fragments = Arc::new(fragments); + manifest + } + + #[test] + fn test_append_matches_the_legacy_path() { + let manifest = manifest_with_fragments(vec![appendable_fragment("data/0.lance")]); + let next = assert_parity( + &manifest, + Operation::Append { + fragments: vec![ + appendable_fragment("data/1.lance"), + appendable_fragment("data/2.lance"), + ], + }, + ); + + // The appended fragments take ids from the manifest's counter, not the + // zero they were handed to the operation with. + assert_eq!( + next.fragments.iter().map(|f| f.id).collect::>(), + vec![0, 1, 2] + ); + } + + #[test] + fn test_append_assigns_stable_row_ids_like_the_legacy_path() { + let mut existing = appendable_fragment("data/1.lance"); + existing.id = 1; + existing.row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&RowIdSequence::from( + 0u64..10, + )))); + let manifest = make_stable_row_id_manifest(vec![existing]); + + let next = assert_parity( + &manifest, + Operation::Append { + fragments: vec![appendable_fragment("data/2.lance")], + }, + ); + + assert_eq!(next.next_row_id, 1010); + let appended = next.fragments.iter().find(|f| f.id == 2).unwrap(); + assert!(appended.row_id_meta.is_some()); + assert!(appended.created_at_version_meta.is_some()); + } + + #[test] + fn test_append_rejects_a_fragment_without_a_row_count() { + let operation = Operation::Append { + fragments: vec![Fragment::new(0)], + }; + let error = Vec::::try_from(&operation).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + } + + #[test] + fn test_delete_matches_the_legacy_path() { + let manifest = manifest_with_fragments(vec![appendable_fragment("data/0.lance"), { + let mut fragment = appendable_fragment("data/1.lance"); + fragment.id = 1; + fragment + }]); + + let mut updated = manifest.fragments[0].clone(); + updated.deletion_file = Some(DeletionFile { + read_version: manifest.version, + id: 7, + file_type: DeletionFileType::Array, + num_deleted_rows: Some(3), + base_id: None, + }); + + let next = assert_parity( + &manifest, + Operation::Delete { + updated_fragments: vec![updated], + deleted_fragment_ids: vec![1], + predicate: "id > 5".into(), + }, + ); + + assert_eq!(next.fragments.len(), 1); + assert!(next.fragments[0].deletion_file.is_some()); + } + + #[test] + fn test_update_bases_matches_the_legacy_path() { + let manifest = manifest_with_fragments(vec![appendable_fragment("data/0.lance")]); + let next = assert_parity( + &manifest, + Operation::UpdateBases { + new_bases: vec![ + BasePath::new(0, "s3://bucket/a".into(), Some("a".into()), false), + BasePath::new(0, "s3://bucket/b".into(), Some("b".into()), false), + ], + }, + ); + + assert_eq!(next.base_paths.len(), 2); + } + + #[test] + fn test_an_untranslated_operation_is_rejected() { + let operation = Operation::ReserveFragments { num_fragments: 3 }; + let error = Vec::::try_from(&operation).unwrap_err(); + assert!(matches!(error, Error::NotSupported { .. }), "{error:?}"); + assert!(error.to_string().contains("ReserveFragments"), "{error}"); + } +} From 40524b8ec7311f1898d4cd891cf6e400fa1c3bfa Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sat, 15 Aug 2026 17:26:32 -0700 Subject: [PATCH 07/24] feat(transaction): translate DataReplacement into actions Replacing a field's data becomes TombstoneFieldData followed by AddDataFile. The legacy path swaps the path on the existing file in place, so the two produce the same set of data files per fragment but not necessarily the same order; files are addressed by field, so the order carries no meaning. Also documents why Merge and Project are not translated: both hand over a whole new schema rather than a delta, and Project needs a field-removal action this draft does not define. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/transaction/action/translate.rs | 121 +++++++++++++++++- 1 file changed, 117 insertions(+), 4 deletions(-) diff --git a/rust/lance-table/src/transaction/action/translate.rs b/rust/lance-table/src/transaction/action/translate.rs index a3a333b64e0..89b7c061360 100644 --- a/rust/lance-table/src/transaction/action/translate.rs +++ b/rust/lance-table/src/transaction/action/translate.rs @@ -11,11 +11,18 @@ //! Translation is fail-closed. An operation whose recipe is not written yet, or //! one carrying a detail the actions cannot express, is rejected rather than //! silently translated into something narrower. +//! +//! `Merge` and `Project` are not translated. Both hand over a whole new schema +//! rather than a description of what changed, so recovering the delta needs the +//! read version's schema to diff against, and `Project` additionally needs a +//! field-removal action that this draft does not define. -use super::UserAction; -use super::{Action, AddBase, AddDataFile, AddFragment, Ref, RemoveFragment, SetDeletionFile}; +use super::{ + Action, AddBase, AddDataFile, AddFragment, Ref, RemoveFragment, SetDeletionFile, + TombstoneFieldData, UserAction, +}; use crate::format::Fragment; -use crate::transaction::Operation; +use crate::transaction::{DataReplacementGroup, Operation}; use lance_core::{Error, Result}; impl TryFrom<&Operation> for Vec { @@ -48,6 +55,10 @@ impl TryFrom<&Operation> for Vec { }) .collect(), )]), + Operation::DataReplacement { replacements } => Ok(vec![UserAction::new( + format!("replace data files in {} fragments", replacements.len()), + data_replacement_actions(replacements)?, + )]), other => Err(Error::not_supported(format!( "translating a {} operation into actions", other.name() @@ -117,6 +128,30 @@ fn delete_actions(updated_fragments: &[Fragment], deleted_fragment_ids: &[u64]) actions } +/// Replacing a field's data is a drop of the old backing file followed by an +/// add of the new one. The legacy path swaps the path on the existing file in +/// place instead, so the resulting fragment holds the same set of data files +/// but not necessarily in the same order -- files are addressed by field, so +/// the order carries no meaning. +fn data_replacement_actions(replacements: &[DataReplacementGroup]) -> Result> { + let mut actions = Vec::with_capacity(replacements.len() * 2); + for DataReplacementGroup(fragment_id, new_file) in replacements { + let fragment = Ref::Committed(*fragment_id); + actions.push(Action::TombstoneFieldData(TombstoneFieldData { + fragment, + field_ids: new_file.fields.to_vec(), + data_change: true, + })); + actions.push(Action::AddDataFile(AddDataFile { + fragment, + file: new_file.clone(), + field_ids: committed_field_refs(new_file.fields.as_ref())?, + data_change: true, + })); + } + Ok(actions) +} + fn committed_field_refs(field_ids: &[i32]) -> Result> { field_ids .iter() @@ -156,7 +191,22 @@ mod tests { Operation::UserOperation(UserOperation::new("translated", actions)), ); - assert_eq!(translated.fragments, legacy.fragments); + // Data files are addressed by field, so the two paths are allowed to + // hold them in different orders within a fragment. + assert_eq!(translated.fragments.len(), legacy.fragments.len()); + for (translated, legacy) in translated.fragments.iter().zip(legacy.fragments.iter()) { + assert_eq!(sorted_files(translated), sorted_files(legacy)); + assert_eq!( + Fragment { + files: Vec::new(), + ..translated.clone() + }, + Fragment { + files: Vec::new(), + ..legacy.clone() + } + ); + } assert_eq!(translated.schema, legacy.schema); assert_eq!(translated.base_paths, legacy.base_paths); assert_eq!(translated.next_row_id, legacy.next_row_id); @@ -165,6 +215,12 @@ mod tests { translated } + fn sorted_files(fragment: &Fragment) -> Vec { + let mut files = fragment.files.clone(); + files.sort_by(|a, b| a.path.cmp(&b.path)); + files + } + fn build(manifest: &Manifest, operation: Operation) -> (Manifest, Vec) { Transaction::new(manifest.version, operation, None) .build_manifest( @@ -289,6 +345,63 @@ mod tests { assert_eq!(next.base_paths.len(), 2); } + #[test] + fn test_data_replacement_matches_the_legacy_path() { + let mut fragment = appendable_fragment("data/0.lance"); + fragment.files.push(DataFile::new( + "data/0b.lance", + vec![1], + vec![0], + 2, + 0, + None, + None, + )); + let manifest = manifest_with_fragments(vec![fragment]); + + let replacement = DataFile::new("data/0-new.lance", vec![0], vec![0], 2, 0, None, None); + let next = assert_parity( + &manifest, + Operation::DataReplacement { + replacements: vec![DataReplacementGroup(0, replacement)], + }, + ); + + let paths = sorted_files(&next.fragments[0]) + .iter() + .map(|file| file.path.clone()) + .collect::>(); + assert_eq!(paths, vec!["data/0-new.lance", "data/0b.lance"]); + } + + #[test] + fn test_data_replacement_of_an_unbacked_field_is_rejected() { + let manifest = manifest_with_fragments(vec![appendable_fragment("data/0.lance")]); + let operation = Operation::DataReplacement { + replacements: vec![DataReplacementGroup( + 0, + DataFile::new("data/0-new.lance", vec![9], vec![0], 2, 0, None, None), + )], + }; + let actions = Vec::::try_from(&operation).unwrap(); + let error = Transaction::new( + manifest.version, + Operation::UserOperation(UserOperation::new("translated", actions)), + None, + ) + .build_manifest( + Some(&manifest), + Vec::new(), + "tx.txn", + &default_build_config(), + ) + .unwrap_err(); + + // The legacy path treats this as an all-NULL column gaining real data; + // the action form has no way to say "drop this if it is there". + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + } + #[test] fn test_an_untranslated_operation_is_rejected() { let operation = Operation::ReserveFragments { num_fragments: 3 }; From f0132179f900a61db8af4900a476ba3a34a1cf26 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sat, 15 Aug 2026 17:28:14 -0700 Subject: [PATCH 08/24] feat(transaction): compute conflict footprints for action sets A footprint is the set of coordinates an action set writes. Two concurrent sets can both commit when neither writes what the other writes -- one structural rule instead of a matrix over operation pairs, so adding an action means saying which coordinates it writes rather than extending an N-by-N table. Only committed coordinates appear. A minted fragment, field, or base has no id in the read version, so two writers minting at the same time never collide. Footprints are derived from the actions at conflict time and never serialized, so a writer cannot pin down what a reader treats as a conflict and the rule can be tightened without a format change. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance-table/src/transaction/action.rs | 3 + .../src/transaction/action/footprint.rs | 290 ++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100644 rust/lance-table/src/transaction/action/footprint.rs diff --git a/rust/lance-table/src/transaction/action.rs b/rust/lance-table/src/transaction/action.rs index 62b1289906a..341fad14671 100644 --- a/rust/lance-table/src/transaction/action.rs +++ b/rust/lance-table/src/transaction/action.rs @@ -27,9 +27,12 @@ //! by libraries that predate it. mod apply; +mod footprint; mod proto; mod translate; +pub use footprint::{Coordinate, Footprint}; + use crate::format::{BasePath, DataFile, DeletionFile, RowIdMeta}; use crate::rowids::version::RowDatasetVersionMeta; use lance_core::datatypes::Field; diff --git a/rust/lance-table/src/transaction/action/footprint.rs b/rust/lance-table/src/transaction/action/footprint.rs new file mode 100644 index 00000000000..8b7bf9ad2f1 --- /dev/null +++ b/rust/lance-table/src/transaction/action/footprint.rs @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! The set of coordinates an action set writes. +//! +//! Two concurrent action sets can both commit when neither writes anything the +//! other writes. This is a structural test over coordinates rather than a +//! matrix over operation pairs, so it stays a single rule as the vocabulary +//! grows -- adding an action means saying which coordinates it writes, not +//! extending an N-by-N table. +//! +//! Footprints are derived from the actions at conflict time and never +//! serialized. A writer cannot pin down what a reader considers a conflict, and +//! the rule can be tightened in a later release without a format change. + +use super::{Action, Ref, UserOperation}; +use std::collections::HashSet; + +/// One thing an action set writes. +/// +/// Only committed coordinates appear. A minted fragment, field, or base has no +/// id in the read version, so no concurrent writer can be naming the same +/// thing; relocation re-resolves it against whatever version wins. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Coordinate { + /// Whether a committed fragment is still part of the dataset. + FragmentExistence(u64), + /// A committed fragment's deletion file. + FragmentDeletions(u64), + /// The data backing one field within one committed fragment. + FieldData { fragment: u64, field: i32 }, + /// A field's definition in the schema. + FieldDefinition(i32), + /// A base path's name, which the manifest requires to be unique. + BaseName(Option), + /// A base path's location, which the manifest requires to be unique. + BaseLocation(String), +} + +impl Coordinate { + /// The fragment this coordinate lives in, if it is fragment-scoped. + fn fragment(&self) -> Option { + match self { + Self::FragmentExistence(id) | Self::FragmentDeletions(id) => Some(*id), + Self::FieldData { fragment, .. } => Some(*fragment), + Self::FieldDefinition(_) | Self::BaseName(_) | Self::BaseLocation(_) => None, + } + } +} + +/// Everything an action set writes, in one set. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Footprint { + writes: HashSet, + /// Fragments this set removes outright. Removing a fragment writes every + /// coordinate inside it, which cannot be enumerated, so it is tracked + /// separately and matched against the other set by fragment id. + removed_fragments: HashSet, +} + +impl Footprint { + pub fn conflicts_with(&self, other: &Self) -> bool { + if !self.writes.is_disjoint(&other.writes) { + return true; + } + self.removes_a_fragment_touched_by(other) || other.removes_a_fragment_touched_by(self) + } + + fn removes_a_fragment_touched_by(&self, other: &Self) -> bool { + self.removed_fragments.iter().any(|removed| { + other + .writes + .iter() + .any(|coordinate| coordinate.fragment() == Some(*removed)) + }) + } + + fn add(&mut self, coordinate: Coordinate) { + self.writes.insert(coordinate); + } + + fn add_field_data(&mut self, fragment: Ref, fields: impl IntoIterator) { + let Some(fragment) = fragment.committed() else { + return; + }; + for field in fields { + self.add(Coordinate::FieldData { fragment, field }); + } + } +} + +impl From<&UserOperation> for Footprint { + fn from(user_operation: &UserOperation) -> Self { + let mut footprint = Self::default(); + for action in user_operation.iter_actions() { + match action { + // Minting actions name nothing that exists in the read version. + Action::AddFragment(_) | Action::AddField(_) => {} + Action::AddBase(action) => { + footprint.add(Coordinate::BaseName(action.base.name.clone())); + footprint.add(Coordinate::BaseLocation(action.base.path.clone())); + } + Action::AddDataFile(action) => footprint.add_field_data( + action.fragment, + action.field_ids.iter().filter_map(|field| { + field.committed().and_then(|id| i32::try_from(id).ok()) + }), + ), + Action::TombstoneFieldData(action) => { + footprint.add_field_data(action.fragment, action.field_ids.iter().copied()) + } + Action::RemoveFragment(action) => { + if let Some(id) = action.fragment.committed() { + footprint.add(Coordinate::FragmentExistence(id)); + footprint.removed_fragments.insert(id); + } + } + Action::SetDeletionFile(action) => { + footprint.add(Coordinate::FragmentDeletions(action.fragment)) + } + Action::AlterField(action) => { + footprint.add(Coordinate::FieldDefinition(action.field)) + } + } + } + footprint + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::{BasePath, DataFile}; + use crate::transaction::action::{ + AddBase, AddDataFile, AddField, AddFragment, AlterField, RemoveFragment, SetDeletionFile, + TombstoneFieldData, UserAction, + }; + use arrow_schema::{DataType, Field as ArrowField}; + use lance_core::datatypes::Field; + use rstest::rstest; + + fn footprint(actions: Vec) -> Footprint { + Footprint::from(&UserOperation::new( + "test", + vec![UserAction::new("step", actions)], + )) + } + + fn add_fragment(local: u32) -> Action { + Action::AddFragment(AddFragment { + local, + physical_rows: 10, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }) + } + + fn add_data_file(fragment: Ref, fields: &[i32]) -> Action { + Action::AddDataFile(AddDataFile { + fragment, + file: DataFile::new_unstarted("data/x.lance", 2, 0), + field_ids: fields + .iter() + .map(|field| Ref::Committed(*field as u64)) + .collect(), + data_change: true, + }) + } + + fn tombstone(fragment: u64, fields: &[i32]) -> Action { + Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Committed(fragment), + field_ids: fields.to_vec(), + data_change: true, + }) + } + + fn remove_fragment(fragment: u64) -> Action { + Action::RemoveFragment(RemoveFragment { + fragment: Ref::Committed(fragment), + data_change: true, + }) + } + + fn set_deletion_file(fragment: u64) -> Action { + Action::SetDeletionFile(SetDeletionFile { + fragment, + deletion_file: None, + data_change: true, + }) + } + + fn add_base(local: u32, name: &str, path: &str) -> Action { + Action::AddBase(AddBase { + local, + base: BasePath::new(0, path.into(), Some(name.into()), false), + }) + } + + #[test] + fn test_minting_actions_write_nothing() { + let minting = footprint(vec![ + add_fragment(0), + Action::AddField(AddField { + local: 1, + parent: None, + def: Field::try_from(ArrowField::new("new", DataType::Int32, true)).unwrap(), + }), + add_data_file(Ref::Local(0), &[0]), + ]); + + // Two writers appending at the same time never collide. + assert!(!minting.conflicts_with(&minting.clone())); + } + + #[rstest] + #[case::same_field_in_same_fragment( + vec![tombstone(0, &[1])], + vec![add_data_file(Ref::Committed(0), &[1])], + true, + )] + #[case::different_fields_in_same_fragment( + vec![tombstone(0, &[1])], + vec![add_data_file(Ref::Committed(0), &[2])], + false, + )] + #[case::same_field_in_different_fragments( + vec![tombstone(0, &[1])], + vec![tombstone(1, &[1])], + false, + )] + #[case::deletions_do_not_collide_with_field_data( + vec![set_deletion_file(0)], + vec![tombstone(0, &[1])], + false, + )] + #[case::concurrent_deletes_of_one_fragment( + vec![set_deletion_file(0)], + vec![set_deletion_file(0)], + true, + )] + #[case::removal_swallows_the_whole_fragment( + vec![remove_fragment(0)], + vec![tombstone(0, &[1])], + true, + )] + #[case::removal_leaves_other_fragments_alone( + vec![remove_fragment(0)], + vec![tombstone(1, &[1])], + false, + )] + #[case::same_field_definition( + vec![Action::AlterField(AlterField { field: 1, name: Some("a".into()), ..Default::default() })], + vec![Action::AlterField(AlterField { field: 1, nullable: Some(true), ..Default::default() })], + true, + )] + #[case::different_field_definitions( + vec![Action::AlterField(AlterField { field: 1, ..Default::default() })], + vec![Action::AlterField(AlterField { field: 2, ..Default::default() })], + false, + )] + #[case::bases_with_the_same_name( + vec![add_base(0, "a", "s3://bucket/one")], + vec![add_base(0, "a", "s3://bucket/two")], + true, + )] + #[case::bases_with_the_same_location( + vec![add_base(0, "a", "s3://bucket/one")], + vec![add_base(0, "b", "s3://bucket/one")], + true, + )] + #[case::unrelated_bases( + vec![add_base(0, "a", "s3://bucket/one")], + vec![add_base(0, "b", "s3://bucket/two")], + false, + )] + fn test_conflicts( + #[case] ours: Vec, + #[case] theirs: Vec, + #[case] expected: bool, + ) { + let ours = footprint(ours); + let theirs = footprint(theirs); + assert_eq!(ours.conflicts_with(&theirs), expected); + // The relation has to hold whichever side is asking. + assert_eq!(theirs.conflicts_with(&ours), expected); + } +} From 9edfb633918b1ad81b9bbab08399744c317b8eb8 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sat, 15 Aug 2026 17:37:43 -0700 Subject: [PATCH 09/24] feat(commit): resolve action-set conflicts by footprint Replaces the conservative always-retry with a footprint comparison. A legacy operation on the other side gets a footprint through its action translation, so an action set can be checked against a concurrent named operation without either needing an entry in the operation-pair matrix. An operation with no translation still falls back to retry. Rebasing an action set onto a newer version is a no-op: its minted ids are allocated when the actions are applied, against whichever manifest they land on, and its committed references name coordinates that do not move. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance/src/io/commit/conflict_resolver.rs | 139 ++++++++++++++++-- 1 file changed, 130 insertions(+), 9 deletions(-) diff --git a/rust/lance/src/io/commit/conflict_resolver.rs b/rust/lance/src/io/commit/conflict_resolver.rs index 3a0f4fbb6e8..03b708f9094 100644 --- a/rust/lance/src/io/commit/conflict_resolver.rs +++ b/rust/lance/src/io/commit/conflict_resolver.rs @@ -16,6 +16,7 @@ use lance_index::mem_wal::{CompactedSsTable, MEM_WAL_INDEX_NAME}; use lance_select::{RowAddrTreeMap, RowSetOps}; use lance_table::format::IndexMetadata; use lance_table::format::overlay::OverlayCoverage; +use lance_table::transaction::action::{Footprint, UserAction, UserOperation}; use lance_table::{format::Fragment, io::deletion::write_deletion_file}; use roaring::RoaringBitmap; use std::{ @@ -90,9 +91,9 @@ impl<'a> TransactionRebase<'a> { | Operation::Clone { .. } | Operation::Restore { .. } | Operation::UpdateBases { .. } - // An action set can modify fragments, but check_action_txn rejects - // any concurrency before the rebase state is consulted, so there is - // nothing to collect yet. + // An action set can modify fragments, but conflicts against it are + // settled by comparing footprints, which are derived from the + // actions rather than from collected rebase state. | Operation::UserOperation(_) => Ok(Self { transaction, affected_rows, @@ -331,9 +332,19 @@ impl<'a> TransactionRebase<'a> { other_transaction: &Transaction, other_version: u64, ) -> Result<()> { - // Conservative until action footprints land: an action-based - // transaction on either side means retry against the newer version. - Err(self.retryable_conflict_err(other_transaction, other_version)) + let (Some(ours), Some(theirs)) = ( + footprint_of(&self.transaction.operation), + footprint_of(&other_transaction.operation), + ) else { + // One side does not decompose into actions yet, so there is nothing + // to compare and the conservative answer stands. + return Err(self.retryable_conflict_err(other_transaction, other_version)); + }; + + if ours.conflicts_with(&theirs) { + return Err(self.retryable_conflict_err(other_transaction, other_version)); + } + Ok(()) } fn check_delete_txn( @@ -1803,9 +1814,10 @@ impl<'a> TransactionRebase<'a> { | Operation::UpdateConfig { .. } | Operation::UpdateMemWalState { .. } | Operation::UpdateBases { .. } - // Rebasing an action set (relocating its minted ids onto the newer - // version) is not implemented yet; check_action_txn rejects before - // this is reached. + // An action set needs no rewriting to move to a newer version: its + // minted ids are allocated when the actions are applied, against + // whichever manifest they land on, and its committed references + // name coordinates that do not move. | Operation::UserOperation(_) => Ok(self.transaction), } } @@ -2317,6 +2329,21 @@ fn overlay_group_coverage(group: &DataOverlayGroup) -> RoaringBitmap { union } +/// The set of coordinates an operation writes, or `None` when it does not +/// decompose into actions yet. +/// +/// A legacy operation gets a footprint through its action translation, so an +/// action set can be compared against a concurrent named operation without +/// either side needing an entry in the operation-pair matrix. +fn footprint_of(operation: &Operation) -> Option { + match operation { + Operation::UserOperation(user_operation) => Some(Footprint::from(user_operation)), + other => Vec::::try_from(other) + .ok() + .map(|actions| Footprint::from(&UserOperation::new(other.name(), actions))), + } +} + fn wrong_operation_err(op: &Operation) -> Error { Error::internal(format!("function called against a wrong operation: {}", op)) } @@ -2335,6 +2362,9 @@ mod tests { use lance_table::format::IndexMetadata; use lance_table::io::deletion::{deletion_file_path, read_deletion_file}; + use lance_table::transaction::action::{ + Action as TxnAction, Ref as ActionRef, RemoveFragment, TombstoneFieldData, + }; use super::*; use crate::dataset::transaction::{DataReplacementGroup, RewriteGroup}; @@ -4237,6 +4267,97 @@ mod tests { assert!(rebase.check_txn(&txn2, 2).is_ok()); } + fn action_txn(actions: Vec) -> Transaction { + Transaction::new_from_version( + 1, + Operation::UserOperation(UserOperation::new( + "test", + vec![UserAction::new("step", actions)], + )), + ) + } + + fn tombstone_txn(fragment: u64, field: i32) -> Transaction { + action_txn(vec![TxnAction::TombstoneFieldData(TombstoneFieldData { + fragment: ActionRef::Committed(fragment), + field_ids: vec![field], + data_change: true, + })]) + } + + /// Assert the verdict holds whichever transaction is the one rebasing. + async fn assert_conflict(dataset: &Dataset, a: Transaction, b: Transaction, expected: bool) { + for (ours, theirs) in [(a.clone(), b.clone()), (b, a)] { + let mut rebase = TransactionRebase::try_new(dataset, ours, None) + .await + .unwrap(); + assert_eq!(rebase.check_txn(&theirs, 2).is_err(), expected); + } + } + + #[tokio::test] + async fn test_action_txns_on_disjoint_coordinates_do_not_conflict() { + let dataset = test_dataset(10, 2).await; + assert_conflict(&dataset, tombstone_txn(0, 0), tombstone_txn(1, 0), false).await; + assert_conflict(&dataset, tombstone_txn(0, 0), tombstone_txn(0, 1), false).await; + } + + #[tokio::test] + async fn test_action_txns_writing_the_same_field_data_conflict() { + let dataset = test_dataset(10, 2).await; + assert_conflict(&dataset, tombstone_txn(0, 0), tombstone_txn(0, 0), true).await; + } + + #[tokio::test] + async fn test_removing_a_fragment_conflicts_with_a_concurrent_delete_on_it() { + let dataset = test_dataset(10, 2).await; + let removal = action_txn(vec![TxnAction::RemoveFragment(RemoveFragment { + fragment: ActionRef::Committed(0), + data_change: true, + })]); + let delete = Transaction::new_from_version( + 1, + Operation::Delete { + updated_fragments: vec![dataset.fragments()[0].clone()], + deleted_fragment_ids: vec![], + predicate: "a > 5".into(), + }, + ); + + assert_conflict(&dataset, removal, delete, true).await; + } + + #[tokio::test] + async fn test_an_action_txn_does_not_conflict_with_a_concurrent_append() { + let dataset = test_dataset(10, 2).await; + let append = Transaction::new_from_version( + 1, + Operation::Append { + fragments: vec![{ + let mut fragment = Fragment::new(0); + fragment.physical_rows = Some(5); + fragment + }], + }, + ); + + // An append only mints; it names nothing the action set could touch. + assert_conflict(&dataset, tombstone_txn(0, 0), append, false).await; + } + + #[tokio::test] + async fn test_an_untranslatable_operation_stays_conservative() { + let dataset = test_dataset(10, 2).await; + let project = Transaction::new_from_version( + 1, + Operation::Project { + schema: dataset.schema().clone(), + }, + ); + + assert_conflict(&dataset, tombstone_txn(0, 0), project, true).await; + } + #[tokio::test] async fn test_add_bases_name_conflict() { let dataset = test_dataset(10, 2).await; From 6d112021c1068d64dd79bc7e5c4818dec62058a4 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sat, 15 Aug 2026 17:38:58 -0700 Subject: [PATCH 10/24] test(transaction): cover relocating an action set onto a newer version Replaying the same actions against the manifest a previous run produced re-resolves their local tokens against the newer counters, so the second run mints different fragment and field ids without any action changing. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/transaction/action/apply.rs | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/rust/lance-table/src/transaction/action/apply.rs b/rust/lance-table/src/transaction/action/apply.rs index dd347668314..2320dd4a96b 100644 --- a/rust/lance-table/src/transaction/action/apply.rs +++ b/rust/lance-table/src/transaction/action/apply.rs @@ -981,6 +981,53 @@ mod tests { ); } + #[test] + fn test_an_action_set_relocates_onto_a_newer_version() { + let actions = vec![ + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("added"), + }), + Action::AddFragment(AddFragment { + local: 0, + physical_rows: 10, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Local(0), + file: DataFile::new_unstarted("data/new.lance", 2, 0), + field_ids: vec![Ref::Local(0)], + data_change: true, + }), + ]; + + // Replaying the very same actions against the manifest the first run + // produced re-resolves both local tokens against the newer counters. + let first = apply(&backed_manifest(), actions.clone()).unwrap(); + let second = apply(&first, actions).unwrap(); + + assert_eq!( + second.fragments.iter().map(|f| f.id).collect::>(), + vec![0, 1, 2] + ); + assert_eq!( + second + .schema + .fields_pre_order() + .map(|field| field.id) + .collect::>(), + vec![0, 1, 2] + ); + // The second run's data file points at the field the second run minted, + // not at the one the first run did. + let relocated = second.fragments.iter().find(|f| f.id == 2).unwrap(); + assert_eq!(relocated.files[0].fields.as_ref(), &[2]); + } + #[test] fn test_action_set_cannot_create_a_dataset() { let transaction = Transaction::new( From b7c7d1b9d767099a2e02fa821c689da32c8a6bad Mon Sep 17 00:00:00 2001 From: Will Jones Date: Sat, 15 Aug 2026 17:45:23 -0700 Subject: [PATCH 11/24] test(commit): end-to-end composite transactions against a real dataset Each test is one commit doing work that would otherwise have taken several: a fragment added and then modified, a field added and then filled, both inside a single version. Also covers row id assignment for minted fragments and both sides of the footprint conflict rule through the real commit path. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance/tests/composite_transaction.rs | 284 ++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 rust/lance/tests/composite_transaction.rs diff --git a/rust/lance/tests/composite_transaction.rs b/rust/lance/tests/composite_transaction.rs new file mode 100644 index 00000000000..0cd9cf232ce --- /dev/null +++ b/rust/lance/tests/composite_transaction.rs @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! End-to-end coverage for committing an action set against a real dataset. +//! +//! Each of these is a single commit that does work a named operation would +//! have needed several commits for: a fragment is added and then modified, a +//! field is added and then filled, all inside one version. That is what the +//! action vocabulary buys -- steps that reference each other's minted ids can +//! be squashed into one atomic manifest change. +//! +//! The commit path checks that a referenced data file exists, so these tests +//! point their actions at files the fixture dataset already wrote. The +//! resulting datasets are inspected through their manifests rather than read -- +//! the files hold the wrong columns for where they end up attached. + +use std::sync::Arc; + +use arrow_array::{Int32Array, RecordBatch}; +use arrow_schema::{DataType, Field, Schema}; +use lance::Dataset; +use lance::dataset::{CommitBuilder, InsertBuilder, WriteParams}; +use lance_table::format::DataFile; +use lance_table::transaction::action::{ + Action, AddDataFile, AddField, AddFragment, Ref, TombstoneFieldData, UserAction, UserOperation, +}; +use lance_table::transaction::{Operation, Transaction}; + +/// A two-fragment dataset, so its two data files can stand in for the files an +/// action set would otherwise have had to write. +async fn test_dataset(enable_stable_row_ids: bool) -> Dataset { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let data = + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from_iter_values(0..10))]).unwrap(); + + InsertBuilder::new("memory://") + .with_params(&WriteParams { + enable_stable_row_ids, + max_rows_per_file: 5, + ..Default::default() + }) + .execute(vec![data]) + .await + .unwrap() +} + +fn existing_data_file(dataset: &Dataset, fragment: usize) -> DataFile { + dataset.fragments()[fragment].files[0].clone() +} + +async fn commit(dataset: Dataset, actions: Vec) -> Dataset { + let read_version = dataset.version().version; + CommitBuilder::new(Arc::new(dataset)) + .execute(Transaction::new( + read_version, + Operation::UserOperation(UserOperation::new( + "composite", + vec![UserAction::new("step", actions)], + )), + None, + )) + .await + .unwrap() +} + +#[tokio::test] +async fn test_one_commit_adds_a_fragment_and_then_modifies_it() { + let dataset = test_dataset(false).await; + let before = dataset.version().version; + let first_file = existing_data_file(&dataset, 0); + let second_file = existing_data_file(&dataset, 1); + let second_path = second_file.path.clone(); + + let dataset = commit( + dataset, + vec![ + Action::AddFragment(AddFragment { + local: 0, + physical_rows: 4, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Local(0), + file: first_file, + field_ids: vec![Ref::Committed(0)], + data_change: true, + }), + // The same commit then replaces the data it just added, naming the + // fragment by the token it was minted under. + Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Local(0), + field_ids: vec![0], + data_change: true, + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Local(0), + file: second_file, + field_ids: vec![Ref::Committed(0)], + data_change: true, + }), + ], + ) + .await; + + assert_eq!(dataset.version().version, before + 1); + let fragments = dataset.fragments(); + assert_eq!(fragments.len(), 3); + let added = fragments.last().unwrap(); + assert_eq!(added.physical_rows, Some(4)); + // The tombstoned file is gone; only the replacement survives the commit. + let paths = added + .files + .iter() + .map(|file| file.path.clone()) + .collect::>(); + assert_eq!(paths, vec![second_path]); +} + +#[tokio::test] +async fn test_one_commit_adds_a_field_and_then_fills_it() { + let dataset = test_dataset(false).await; + let fragment_id = dataset.fragments()[0].id; + let file = existing_data_file(&dataset, 1); + let path = file.path.clone(); + + let dataset = commit( + dataset, + vec![ + Action::AddField(AddField { + local: 0, + parent: None, + def: lance_core::datatypes::Field::try_from(Field::new("b", DataType::Int32, true)) + .unwrap(), + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Committed(fragment_id), + file, + field_ids: vec![Ref::Local(0)], + data_change: true, + }), + ], + ) + .await; + + let field = dataset.schema().field("b").expect("field b was added"); + let fragment = &dataset.fragments()[0]; + let added = fragment + .files + .iter() + .find(|file| file.path == path) + .expect("the new field's data file was attached"); + // The file points at the id the commit minted, which the caller never knew. + assert_eq!(added.fields.as_ref(), &[field.id]); +} + +#[tokio::test] +async fn test_one_commit_assigns_row_ids_to_the_fragments_it_mints() { + let dataset = test_dataset(true).await; + let next_row_id = dataset.manifest().next_row_id; + assert_eq!(next_row_id, 10); + + let dataset = commit( + dataset, + vec![ + Action::AddFragment(AddFragment { + local: 0, + physical_rows: 4, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }), + Action::AddFragment(AddFragment { + local: 1, + physical_rows: 6, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }), + ], + ) + .await; + + assert_eq!(dataset.manifest().next_row_id, 20); + for fragment in dataset.fragments().iter().skip(2) { + assert!( + fragment.row_id_meta.is_some(), + "fragment {} was minted without row ids", + fragment.id + ); + assert!(fragment.created_at_version_meta.is_some()); + } +} + +#[tokio::test] +async fn test_two_action_sets_on_disjoint_coordinates_both_commit() { + let dataset = Arc::new(test_dataset(false).await); + let read_version = dataset.version().version; + + let append = |local| { + vec![Action::AddFragment(AddFragment { + local, + physical_rows: 4, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + })] + }; + + let first = CommitBuilder::new(dataset.clone()) + .execute(Transaction::new( + read_version, + Operation::UserOperation(UserOperation::new( + "first", + vec![UserAction::new("step", append(0))], + )), + None, + )) + .await + .unwrap(); + + // The second commit still reads the original version, so it has to be + // checked against the first. Both only mint, so neither writes anything + // the other does. + let second = CommitBuilder::new(dataset) + .execute(Transaction::new( + read_version, + Operation::UserOperation(UserOperation::new( + "second", + vec![UserAction::new("step", append(0))], + )), + None, + )) + .await + .unwrap(); + + assert_eq!(second.version().version, first.version().version + 1); + assert_eq!(second.fragments().len(), 4); +} + +#[tokio::test] +async fn test_two_action_sets_writing_the_same_field_data_conflict() { + let dataset = Arc::new(test_dataset(false).await); + let read_version = dataset.version().version; + let fragment_id = dataset.fragments()[0].id; + + let tombstone = || { + Transaction::new( + read_version, + Operation::UserOperation(UserOperation::new( + "tombstone", + vec![UserAction::new( + "step", + vec![Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Committed(fragment_id), + field_ids: vec![0], + data_change: true, + })], + )], + )), + None, + ) + }; + + CommitBuilder::new(dataset.clone()) + .execute(tombstone()) + .await + .unwrap(); + + let error = CommitBuilder::new(dataset) + .with_max_retries(0) + .execute(tombstone()) + .await + .unwrap_err(); + assert!( + error.to_string().contains("preempted"), + "unexpected error: {error}" + ); +} From 993e53f1aaee9a600de680803f6f045f1a72b7f4 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 17 Aug 2026 12:59:53 -0700 Subject: [PATCH 12/24] feat(transaction): add the DropField action Removes a field from the schema, taking its descendants with it. The slots that backed the dropped fields in each data file are tombstoned rather than removed, so a file's remaining columns stay at the positions they were written at; a file left backing nothing live is pruned during normalization. For conflicts, a drop writes every coordinate belonging to the field -- its definition and its data in every fragment -- so it collides with a concurrent alter or data rewrite of the same field. Field ids come from a monotonic counter, so a dropped id is never reused and a stale data file naming it cannot be mistaken for a later field. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance-table/src/transaction/action.rs | 16 +- .../src/transaction/action/apply.rs | 165 +++++++++++++++++- .../src/transaction/action/footprint.rs | 66 ++++++- .../src/transaction/action/proto.rs | 25 ++- .../src/transaction/action/translate.rs | 6 +- rust/lance/tests/composite_transaction.rs | 38 +++- 6 files changed, 299 insertions(+), 17 deletions(-) diff --git a/rust/lance-table/src/transaction/action.rs b/rust/lance-table/src/transaction/action.rs index 341fad14671..317d0980c7e 100644 --- a/rust/lance-table/src/transaction/action.rs +++ b/rust/lance-table/src/transaction/action.rs @@ -139,6 +139,7 @@ pub enum Action { RemoveFragment(RemoveFragment), SetDeletionFile(SetDeletionFile), AlterField(AlterField), + DropField(DropField), } impl Action { @@ -152,6 +153,7 @@ impl Action { Self::RemoveFragment(_) => "RemoveFragment", Self::SetDeletionFile(_) => "SetDeletionFile", Self::AlterField(_) => "AlterField", + Self::DropField(_) => "DropField", } } @@ -167,7 +169,9 @@ impl Action { Self::TombstoneFieldData(action) => action.data_change, Self::RemoveFragment(action) => action.data_change, Self::SetDeletionFile(action) => action.data_change, - // Schema and base-path changes touch no row values. + // Dropping a field discards the values it held. + Self::DropField(_) => true, + // Other schema and base-path changes touch no row values. Self::AddField(_) | Self::AddBase(_) | Self::AlterField(_) => false, } } @@ -299,6 +303,16 @@ pub struct AlterField { pub nullable: Option, } +/// Remove a field from the schema. +/// +/// The field's descendants go with it, since a struct's children cannot outlive +/// it. At apply, any data file left backing no live field is dropped, and any +/// index over a removed field is discarded. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct DropField { + pub field: i32, +} + #[cfg(test)] mod tests { use super::*; diff --git a/rust/lance-table/src/transaction/action/apply.rs b/rust/lance-table/src/transaction/action/apply.rs index 2320dd4a96b..fa1cee8a135 100644 --- a/rust/lance-table/src/transaction/action/apply.rs +++ b/rust/lance-table/src/transaction/action/apply.rs @@ -11,8 +11,8 @@ //! different ids without any of the actions changing. use super::{ - Action, AddBase, AddDataFile, AddField, AddFragment, AlterField, Ref, RemoveFragment, - SetDeletionFile, TombstoneFieldData, UserOperation, + Action, AddBase, AddDataFile, AddField, AddFragment, AlterField, DropField, Ref, + RemoveFragment, SetDeletionFile, TombstoneFieldData, UserOperation, }; use crate::format::{BasePath, Fragment, IndexMetadata, Manifest, ManifestBuildConfig}; use crate::rowids::version::build_version_meta; @@ -157,6 +157,7 @@ impl ApplyState { Action::RemoveFragment(action) => self.remove_fragment(action), Action::SetDeletionFile(action) => self.set_deletion_file(action), Action::AlterField(action) => self.alter_field(action), + Action::DropField(action) => self.drop_field(action), } } @@ -365,6 +366,54 @@ impl ApplyState { Ok(()) } + fn drop_field(&mut self, action: &DropField) -> Result<()> { + let field = self.schema.field_by_id(action.field).ok_or_else(|| { + Error::invalid_input(format!( + "DropField names field {}, which does not exist", + action.field + )) + })?; + // A struct's children cannot outlive it, so the whole subtree goes. + let mut dropped = HashSet::new(); + collect_subtree_ids(field, &mut dropped); + remove_field(&mut self.schema.fields, action.field); + + // The fields are gone from the schema, so the slots that backed them in + // each data file are dead. Tombstoning rather than rewriting the field + // list keeps a file's remaining columns at the positions they were + // written at; a file left with nothing live is pruned during + // normalization. + for fragment in self.fragments.iter_mut() { + for file in fragment.files.iter_mut() { + if !file.fields.iter().any(|id| dropped.contains(id)) { + continue; + } + let fields = file + .fields + .iter() + .map(|id| { + if dropped.contains(id) { + TOMBSTONED_FIELD + } else { + *id + } + }) + .collect::>(); + file.fields = fields.into(); + } + + let overlaid: Vec = dropped + .iter() + .filter_map(|id| u32::try_from(*id).ok()) + .collect(); + crate::format::overlay::tombstone_overlay_fields(&mut fragment.overlays, &overlaid); + } + + // Indices over a field that no longer exists are discarded wholesale by + // `retain_relevant_indices`, so there is nothing to record here. + Ok(()) + } + /// Stamp row ids and version metadata onto the fragments this operation /// minted, matching what an Append does for its new fragments. fn assign_row_ids_to_minted_fragments( @@ -427,6 +476,25 @@ impl ApplyState { } } +fn collect_subtree_ids(field: &Field, out: &mut HashSet) { + out.insert(field.id); + for child in &field.children { + collect_subtree_ids(child, out); + } +} + +/// Remove the field with `field_id` from `fields`, at whatever depth it sits. +fn remove_field(fields: &mut Vec, field_id: i32) { + let before = fields.len(); + fields.retain(|field| field.id != field_id); + if fields.len() != before { + return; + } + for field in fields.iter_mut() { + remove_field(&mut field.children, field_id); + } +} + fn fragment_mut<'a>( fragments: &'a mut [Fragment], fragment_id: u64, @@ -783,6 +851,99 @@ mod tests { assert!(error.to_string().contains("field 7"), "{error}"); } + #[test] + fn test_drop_field_removes_it_and_its_data() { + let manifest = backed_manifest(); + let (next, indices) = apply_with_indices( + &manifest, + vec![Action::DropField(DropField { field: 0 })], + vec![sample_index_metadata("idx")], + ) + .unwrap(); + + assert!(next.schema.field_by_id(0).is_none()); + // The file backed only the dropped field, so nothing is left of it. + assert!(next.fragments[0].files.is_empty()); + // An index over a field that no longer exists is discarded outright. + assert!(indices.is_empty()); + } + + #[test] + fn test_drop_field_keeps_a_file_with_a_surviving_field() { + let mut manifest = backed_manifest(); + let mut schema_field = added_field("keep"); + schema_field.id = 1; + manifest.schema.fields.push(schema_field); + let mut fragment = manifest.fragments[0].clone(); + fragment.files[0] = DataFile::new("data/0.lance", vec![0, 1], vec![0, 1], 2, 0, None, None); + manifest.fragments = Arc::new(vec![fragment]); + + let next = apply(&manifest, vec![Action::DropField(DropField { field: 0 })]).unwrap(); + + assert!(next.schema.field_by_id(0).is_none()); + assert!(next.schema.field_by_id(1).is_some()); + // The surviving field stays at the position it was written at. + assert_eq!( + next.fragments[0].files[0].fields.as_ref(), + &[TOMBSTONED_FIELD, 1] + ); + } + + #[test] + fn test_drop_field_takes_the_whole_subtree() { + let mut manifest = backed_manifest(); + let mut parent = Field::try_from(ArrowField::new("parent", DataType::Int32, true)).unwrap(); + parent.id = 1; + let mut child = added_field("child"); + child.id = 2; + child.parent_id = 1; + parent.children.push(child); + manifest.schema.fields.push(parent); + + let next = apply(&manifest, vec![Action::DropField(DropField { field: 1 })]).unwrap(); + + assert!(next.schema.field_by_id(1).is_none()); + assert!( + next.schema.field_by_id(2).is_none(), + "a struct's children cannot outlive it" + ); + } + + #[test] + fn test_drop_field_rejects_a_missing_field() { + let manifest = backed_manifest(); + let error = apply(&manifest, vec![Action::DropField(DropField { field: 7 })]).unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + assert!(error.to_string().contains("field 7"), "{error}"); + } + + #[test] + fn test_drop_field_then_add_a_field_reuses_no_id() { + let manifest = backed_manifest(); + let next = apply( + &manifest, + vec![ + Action::DropField(DropField { field: 0 }), + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("replacement"), + }), + ], + ) + .unwrap(); + + // Field ids come from a monotonic counter, never from the freed id -- + // an old data file naming id 0 must not be read as the new field. + let ids = next + .schema + .fields_pre_order() + .map(|field| field.id) + .collect::>(); + assert_eq!(ids, vec![1]); + } + #[test] fn test_add_fragment_and_data_file_mint_ids() { let manifest = sample_manifest(); diff --git a/rust/lance-table/src/transaction/action/footprint.rs b/rust/lance-table/src/transaction/action/footprint.rs index 8b7bf9ad2f1..3eb2a3d7b18 100644 --- a/rust/lance-table/src/transaction/action/footprint.rs +++ b/rust/lance-table/src/transaction/action/footprint.rs @@ -46,6 +46,18 @@ impl Coordinate { Self::FieldDefinition(_) | Self::BaseName(_) | Self::BaseLocation(_) => None, } } + + /// The field this coordinate belongs to, if it is field-scoped. + fn field(&self) -> Option { + match self { + Self::FieldData { field, .. } => Some(*field), + Self::FieldDefinition(id) => Some(*id), + Self::FragmentExistence(_) + | Self::FragmentDeletions(_) + | Self::BaseName(_) + | Self::BaseLocation(_) => None, + } + } } /// Everything an action set writes, in one set. @@ -56,6 +68,15 @@ pub struct Footprint { /// coordinate inside it, which cannot be enumerated, so it is tracked /// separately and matched against the other set by fragment id. removed_fragments: HashSet, + /// Fields this set drops from the schema. Like a fragment removal, this + /// writes every coordinate belonging to the field -- its definition and its + /// data in every fragment -- so it is matched by field id. + /// + /// Only the named field, not its descendants: a footprint has no schema to + /// expand a struct with. A concurrent write to a child of a dropped struct + /// is therefore not caught here and fails when it is applied against the + /// version where the child no longer exists. + removed_fields: HashSet, } impl Footprint { @@ -63,15 +84,18 @@ impl Footprint { if !self.writes.is_disjoint(&other.writes) { return true; } - self.removes_a_fragment_touched_by(other) || other.removes_a_fragment_touched_by(self) + self.removes_something_touched_by(other) || other.removes_something_touched_by(self) } - fn removes_a_fragment_touched_by(&self, other: &Self) -> bool { - self.removed_fragments.iter().any(|removed| { - other - .writes - .iter() - .any(|coordinate| coordinate.fragment() == Some(*removed)) + /// Whether this set removes a fragment or field that `other` also writes to. + fn removes_something_touched_by(&self, other: &Self) -> bool { + other.writes.iter().any(|coordinate| { + coordinate + .fragment() + .is_some_and(|id| self.removed_fragments.contains(&id)) + || coordinate + .field() + .is_some_and(|id| self.removed_fields.contains(&id)) }) } @@ -121,6 +145,10 @@ impl From<&UserOperation> for Footprint { Action::AlterField(action) => { footprint.add(Coordinate::FieldDefinition(action.field)) } + Action::DropField(action) => { + footprint.add(Coordinate::FieldDefinition(action.field)); + footprint.removed_fields.insert(action.field); + } } } footprint @@ -132,8 +160,8 @@ mod tests { use super::*; use crate::format::{BasePath, DataFile}; use crate::transaction::action::{ - AddBase, AddDataFile, AddField, AddFragment, AlterField, RemoveFragment, SetDeletionFile, - TombstoneFieldData, UserAction, + AddBase, AddDataFile, AddField, AddFragment, AlterField, DropField, RemoveFragment, + SetDeletionFile, TombstoneFieldData, UserAction, }; use arrow_schema::{DataType, Field as ArrowField}; use lance_core::datatypes::Field; @@ -261,6 +289,26 @@ mod tests { vec![Action::AlterField(AlterField { field: 2, ..Default::default() })], false, )] + #[case::dropping_a_field_collides_with_altering_it( + vec![Action::DropField(DropField { field: 1 })], + vec![Action::AlterField(AlterField { field: 1, nullable: Some(true), ..Default::default() })], + true, + )] + #[case::dropping_a_field_collides_with_rewriting_its_data( + vec![Action::DropField(DropField { field: 1 })], + vec![tombstone(0, &[1])], + true, + )] + #[case::dropping_a_field_leaves_other_fields_alone( + vec![Action::DropField(DropField { field: 1 })], + vec![tombstone(0, &[2])], + false, + )] + #[case::dropping_a_field_leaves_deletions_alone( + vec![Action::DropField(DropField { field: 1 })], + vec![set_deletion_file(0)], + false, + )] #[case::bases_with_the_same_name( vec![add_base(0, "a", "s3://bucket/one")], vec![add_base(0, "a", "s3://bucket/two")], diff --git a/rust/lance-table/src/transaction/action/proto.rs b/rust/lance-table/src/transaction/action/proto.rs index 4dd4d320711..ef11b5ea8a4 100644 --- a/rust/lance-table/src/transaction/action/proto.rs +++ b/rust/lance-table/src/transaction/action/proto.rs @@ -9,8 +9,8 @@ //! must abort the commit rather than be treated as a no-op. use super::{ - Action, AddBase, AddDataFile, AddField, AddFragment, AlterField, Ref, RemoveFragment, - SetDeletionFile, TombstoneFieldData, UserAction, UserOperation, + Action, AddBase, AddDataFile, AddField, AddFragment, AlterField, DropField, Ref, + RemoveFragment, SetDeletionFile, TombstoneFieldData, UserAction, UserOperation, }; use crate::format::pb; use crate::format::{BasePath, DataFile, DeletionFile, ExternalFile, RowIdMeta}; @@ -127,6 +127,7 @@ impl From<&Action> for pb::Action { Action::RemoveFragment(action) => pb::action::Action::RemoveFragment(action.into()), Action::SetDeletionFile(action) => pb::action::Action::SetDeletionFile(action.into()), Action::AlterField(action) => pb::action::Action::AlterField(action.into()), + Action::DropField(action) => pb::action::Action::DropField(action.into()), }; Self { action: Some(action), @@ -159,6 +160,7 @@ impl TryFrom for Action { Some(pb::action::Action::AlterField(action)) => { Ok(Self::AlterField(action.try_into()?)) } + Some(pb::action::Action::DropField(action)) => Ok(Self::DropField(action.try_into()?)), // The drafted vocabulary is larger than what is implemented. Reject // rather than skip: silently dropping an action would apply a // partial transaction. @@ -442,6 +444,24 @@ impl TryFrom for AlterField { } } +impl From<&DropField> for pb::DropField { + fn from(value: &DropField) -> Self { + Self { + field: value.field as u64, + } + } +} + +impl TryFrom for DropField { + type Error = Error; + + fn try_from(message: pb::DropField) -> Result { + Ok(Self { + field: field_id_from_wire(message.field)?, + }) + } +} + fn required(value: Option, what: &str) -> Result { value.ok_or_else(|| Error::invalid_input(format!("{what} is required but was not set"))) } @@ -510,6 +530,7 @@ mod tests { logical_type: Some("int64".into()), nullable: Some(false), }), + Action::DropField(DropField { field: 3 }), ] } diff --git a/rust/lance-table/src/transaction/action/translate.rs b/rust/lance-table/src/transaction/action/translate.rs index 89b7c061360..be991de5557 100644 --- a/rust/lance-table/src/transaction/action/translate.rs +++ b/rust/lance-table/src/transaction/action/translate.rs @@ -14,8 +14,10 @@ //! //! `Merge` and `Project` are not translated. Both hand over a whole new schema //! rather than a description of what changed, so recovering the delta needs the -//! read version's schema to diff against, and `Project` additionally needs a -//! field-removal action that this draft does not define. +//! read version's schema to diff against -- which this conversion, taking only +//! the operation, does not have. The actions themselves are sufficient: +//! `Project` is a set of [`DropField`](super::DropField)s and `Merge` a set of +//! [`AddField`](super::AddField)s plus their data files. use super::{ Action, AddBase, AddDataFile, AddFragment, Ref, RemoveFragment, SetDeletionFile, diff --git a/rust/lance/tests/composite_transaction.rs b/rust/lance/tests/composite_transaction.rs index 0cd9cf232ce..7d967cd012f 100644 --- a/rust/lance/tests/composite_transaction.rs +++ b/rust/lance/tests/composite_transaction.rs @@ -22,7 +22,8 @@ use lance::Dataset; use lance::dataset::{CommitBuilder, InsertBuilder, WriteParams}; use lance_table::format::DataFile; use lance_table::transaction::action::{ - Action, AddDataFile, AddField, AddFragment, Ref, TombstoneFieldData, UserAction, UserOperation, + Action, AddDataFile, AddField, AddFragment, DropField, Ref, TombstoneFieldData, UserAction, + UserOperation, }; use lance_table::transaction::{Operation, Transaction}; @@ -156,6 +157,41 @@ async fn test_one_commit_adds_a_field_and_then_fills_it() { assert_eq!(added.fields.as_ref(), &[field.id]); } +#[tokio::test] +async fn test_one_commit_swaps_a_field_for_a_new_one() { + let dataset = test_dataset(false).await; + let fragment_id = dataset.fragments()[0].id; + let file = existing_data_file(&dataset, 1); + + let dataset = commit( + dataset, + vec![ + Action::DropField(DropField { field: 0 }), + Action::AddField(AddField { + local: 0, + parent: None, + def: lance_core::datatypes::Field::try_from(Field::new("a", DataType::Int64, true)) + .unwrap(), + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Committed(fragment_id), + file, + field_ids: vec![Ref::Local(0)], + data_change: true, + }), + ], + ) + .await; + + // Dropping "a" and adding a new "a" of a different type is one version, and + // the new field gets a fresh id rather than inheriting the dropped one. + let schema = dataset.schema(); + assert_eq!(schema.fields.len(), 1); + let field = schema.field("a").unwrap(); + assert_ne!(field.id, 0); + assert_eq!(field.logical_type.to_string(), "int64"); +} + #[tokio::test] async fn test_one_commit_assigns_row_ids_to_the_fragments_it_mints() { let dataset = test_dataset(true).await; From dc0ae2bf4ef444785fb2cd423fe1da17cb60ac8e Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 17 Aug 2026 14:22:29 -0700 Subject: [PATCH 13/24] refactor(transaction): give each action its own module The action code was split by phase -- apply, footprint, proto -- so understanding one action meant reading four files and adding one meant touching four match statements. Each action now owns a module holding its definition, `apply`, `footprint`, wire conversions, and tests. `Action` dispatches to them. The phase modules keep what is genuinely shared: `apply` holds the `ApplyState` the actions program against, `footprint` the coordinate space and the conflict comparison, `proto` the envelope and dispatch. No behavior change. The one difference is that `AddField` and `AddBase` now check for a duplicate local token before bumping the id counter, matching `AddFragment`; a duplicate token aborts the whole apply either way. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance-table/src/transaction/action.rs | 202 ++-- .../src/transaction/action/add_base.rs | 128 +++ .../src/transaction/action/add_data_file.rs | 162 +++ .../src/transaction/action/add_field.rs | 231 ++++ .../src/transaction/action/add_fragment.rs | 205 ++++ .../src/transaction/action/alter_field.rs | 151 +++ .../src/transaction/action/apply.rs | 992 ++---------------- .../src/transaction/action/drop_field.rs | 223 ++++ .../src/transaction/action/footprint.rs | 57 +- .../src/transaction/action/proto.rs | 348 +----- .../src/transaction/action/remove_fragment.rs | 123 +++ .../transaction/action/set_deletion_file.rs | 126 +++ .../src/transaction/action/test_support.rs | 56 + .../action/tombstone_field_data.rs | 167 +++ 14 files changed, 1805 insertions(+), 1366 deletions(-) create mode 100644 rust/lance-table/src/transaction/action/add_base.rs create mode 100644 rust/lance-table/src/transaction/action/add_data_file.rs create mode 100644 rust/lance-table/src/transaction/action/add_field.rs create mode 100644 rust/lance-table/src/transaction/action/add_fragment.rs create mode 100644 rust/lance-table/src/transaction/action/alter_field.rs create mode 100644 rust/lance-table/src/transaction/action/drop_field.rs create mode 100644 rust/lance-table/src/transaction/action/remove_fragment.rs create mode 100644 rust/lance-table/src/transaction/action/set_deletion_file.rs create mode 100644 rust/lance-table/src/transaction/action/test_support.rs create mode 100644 rust/lance-table/src/transaction/action/tombstone_field_data.rs diff --git a/rust/lance-table/src/transaction/action.rs b/rust/lance-table/src/transaction/action.rs index 317d0980c7e..b4835ce9796 100644 --- a/rust/lance-table/src/transaction/action.rs +++ b/rust/lance-table/src/transaction/action.rs @@ -14,10 +14,17 @@ //! Only the subset of the drafted vocabulary that is implemented appears here -- //! an action this build does not know is rejected on load rather than skipped. //! +//! Each action lives in its own module and owns everything about itself: its +//! definition, how it is applied, which coordinates it writes, and its wire +//! encoding. This module holds the shared vocabulary and dispatches to them. +//! //! ```text -//! action the vocabulary (this module) -//! action::apply applying an action set to produce the next manifest -//! action::proto its persisted protobuf encoding +//! action the vocabulary and the dispatch (this module) +//! action:: one action, end to end +//! action::apply the working state an action set is applied against +//! action::footprint comparing two action sets for conflicts +//! action::proto the envelope around the per-action encodings +//! action::translate lowering a named operation into actions //! ``` //! //! # Stability @@ -26,16 +33,36 @@ //! contract, and a transaction carrying a [`UserOperation`] is rejected outright //! by libraries that predate it. +mod add_base; +mod add_data_file; +mod add_field; +mod add_fragment; +mod alter_field; mod apply; +mod drop_field; mod footprint; mod proto; +mod remove_fragment; +mod set_deletion_file; +mod tombstone_field_data; mod translate; +#[cfg(test)] +mod test_support; + +pub use add_base::AddBase; +pub use add_data_file::AddDataFile; +pub use add_field::AddField; +pub use add_fragment::AddFragment; +pub use alter_field::AlterField; +pub use drop_field::DropField; pub use footprint::{Coordinate, Footprint}; +pub use remove_fragment::RemoveFragment; +pub use set_deletion_file::SetDeletionFile; +pub use tombstone_field_data::TombstoneFieldData; -use crate::format::{BasePath, DataFile, DeletionFile, RowIdMeta}; -use crate::rowids::version::RowDatasetVersionMeta; -use lance_core::datatypes::Field; +use apply::ApplyState; +use lance_core::Result; use lance_core::deepsize::DeepSizeOf; /// A reference to a counter-allocated identifier -- a field id, fragment id, or @@ -128,7 +155,8 @@ impl UserAction { /// A single granular change to the manifest. /// /// The drafted vocabulary is larger than this; the variants here are the ones -/// this build implements end to end. +/// this build implements end to end. Each one is defined, applied, and encoded +/// in the module named after it. #[derive(Debug, Clone, PartialEq, DeepSizeOf)] pub enum Action { AddFragment(AddFragment), @@ -175,6 +203,36 @@ impl Action { Self::AddField(_) | Self::AddBase(_) | Self::AlterField(_) => false, } } + + /// Fold this action into the state the next manifest is built from. + fn apply(&self, state: &mut ApplyState) -> Result<()> { + match self { + Self::AddFragment(action) => action.apply(state), + Self::AddDataFile(action) => action.apply(state), + Self::AddField(action) => action.apply(state), + Self::AddBase(action) => action.apply(state), + Self::TombstoneFieldData(action) => action.apply(state), + Self::RemoveFragment(action) => action.apply(state), + Self::SetDeletionFile(action) => action.apply(state), + Self::AlterField(action) => action.apply(state), + Self::DropField(action) => action.apply(state), + } + } + + /// Record the coordinates this action writes. + fn footprint(&self, footprint: &mut Footprint) { + match self { + Self::AddFragment(action) => action.footprint(footprint), + Self::AddDataFile(action) => action.footprint(footprint), + Self::AddField(action) => action.footprint(footprint), + Self::AddBase(action) => action.footprint(footprint), + Self::TombstoneFieldData(action) => action.footprint(footprint), + Self::RemoveFragment(action) => action.footprint(footprint), + Self::SetDeletionFile(action) => action.footprint(footprint), + Self::AlterField(action) => action.footprint(footprint), + Self::DropField(action) => action.footprint(footprint), + } + } } impl std::fmt::Display for Action { @@ -183,136 +241,6 @@ impl std::fmt::Display for Action { } } -/// Mint a new, empty fragment. -/// -/// Its data files arrive via [`AddDataFile`] actions naming this fragment's -/// local token. A freshly-minted fragment has no deletion vector: it has no -/// committed rows to delete yet. -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub struct AddFragment { - /// Token standing in for the fragment id until it is allocated at apply. - pub local: u32, - /// Physical rows in the fragment, including rows later tombstoned. - pub physical_rows: u64, - /// Stable row id sequence. `None` on datasets without stable row ids, and - /// on datasets that have them but where the ids are assigned at apply. - pub row_id_meta: Option, - /// Per-row version metadata, carried exactly as on - /// [`Fragment`](crate::format::Fragment). `None` means "stamp at apply". - pub last_updated_at_version_meta: Option, - pub created_at_version_meta: Option, - /// `false` marks a pure rearrangement, e.g. a compaction rewrite. - pub data_change: bool, -} - -/// Add a data file to a fragment. -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub struct AddDataFile { - /// The fragment to add the file to: committed, or a fragment minted earlier - /// in the same operation. - pub fragment: Ref, - /// The file. Its `fields` are placeholders and are stamped in at apply from - /// `field_ids`, which is the authority for the column -> field mapping. - pub file: DataFile, - /// One entry per column in `file`, in column order. - pub field_ids: Vec, - /// See [`AddFragment::data_change`]. - pub data_change: bool, -} - -/// Mint a new schema field. -/// -/// A nested column that introduces several fields is several ordered -/// `AddField`s -- parent first, each child naming its parent's local token. -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub struct AddField { - /// Token standing in for the field id until it is allocated at apply. - pub local: u32, - /// The parent field, or `None` for a top-level column. - pub parent: Option, - /// The field definition. Its `id`, `parent_id`, and `children` are ignored: - /// `local` and `parent` carry that structure, and each child is its own - /// action. - pub def: Field, -} - -/// Mint a new base path. -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub struct AddBase { - /// Token standing in for the base id until it is allocated at apply. - pub local: u32, - /// The base path. Its `id` is ignored and stamped in at apply. - pub base: BasePath, -} - -/// Tombstone the data-file binding of committed fields within one fragment. -/// -/// Each field's slot in whatever file currently backs it is marked tombstoned, -/// and a file left with no live field is pruned at apply. Data files have no id -/// of their own and a live field is backed by exactly one file, so this is how a -/// column's data is dropped or superseded: re-encoding a column is a tombstone -/// followed by an [`AddDataFile`] for the same field. -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub struct TombstoneFieldData { - pub fragment: Ref, - /// Committed field ids whose current backing is tombstoned. - pub field_ids: Vec, - /// See [`AddFragment::data_change`]. - pub data_change: bool, -} - -/// Remove a fragment entirely -- every row deleted, or the fragment replaced by -/// compaction. -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub struct RemoveFragment { - pub fragment: Ref, - /// See [`AddFragment::data_change`]. - pub data_change: bool, -} - -/// Set (replace) a fragment's deletion file. -/// -/// This is reference-stable rather than a delta: the fragment id is committed -/// and physical row offsets never move, so the post-image is unambiguous. The -/// newly-deleted rows -- the delta rebase and conflict detection need -- are -/// derived by diffing against the read version rather than serialized. -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub struct SetDeletionFile { - /// The fragment, by committed id. Unlike its sibling fragment actions this - /// takes no [`Ref`]: a fragment minted in the same operation has no - /// committed rows to delete. - pub fragment: u64, - /// The new deletion file, or `None` to clear the fragment's deletions. - pub deletion_file: Option, - /// See [`AddFragment::data_change`]. - pub data_change: bool, -} - -/// Alter facets of an existing field in place, preserving its id. -/// -/// Each facet is independently optional -- present means "change this", absent -/// means "leave it alone" -- so a widening cast and a nullability relaxation on -/// the same field commute. A cast additionally needs a [`TombstoneFieldData`] -/// plus a fresh [`AddDataFile`] to rewrite the data. -#[derive(Debug, Clone, PartialEq, DeepSizeOf, Default)] -pub struct AlterField { - pub field: i32, - pub name: Option, - /// The new Arrow logical type. The cast. - pub logical_type: Option, - pub nullable: Option, -} - -/// Remove a field from the schema. -/// -/// The field's descendants go with it, since a struct's children cannot outlive -/// it. At apply, any data file left backing no live field is dropped, and any -/// index over a removed field is discarded. -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub struct DropField { - pub field: i32, -} - #[cfg(test)] mod tests { use super::*; diff --git a/rust/lance-table/src/transaction/action/add_base.rs b/rust/lance-table/src/transaction/action/add_base.rs new file mode 100644 index 00000000000..d82f163b965 --- /dev/null +++ b/rust/lance-table/src/transaction/action/add_base.rs @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Mint a new base path. + +use super::apply::ApplyState; +use super::proto::required; +use super::{Coordinate, Footprint}; +use crate::format::{BasePath, pb}; +use lance_core::deepsize::DeepSizeOf; +use lance_core::{Error, Result}; + +/// Mint a new base path. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct AddBase { + /// Token standing in for the base id until it is allocated at apply. + pub local: u32, + /// The base path. Its `id` is ignored and stamped in at apply. + pub base: BasePath, +} + +impl AddBase { + pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { + let id = state.mint_base(self.local)?; + + let conflicting = state + .bases() + .find(|base| base.name == self.base.name || base.path == self.base.path); + if let Some(conflicting) = conflicting { + return Err(Error::invalid_input(format!( + "Conflict detected: Base path with name '{:?}' or path '{}' already exists. \ + Existing: name='{:?}', path='{}'", + self.base.name, self.base.path, conflicting.name, conflicting.path + ))); + } + + let mut base = self.base.clone(); + base.id = id; + state.push_base(base); + Ok(()) + } + + /// The base id is minted, but the name and location are not: the manifest + /// requires both to be unique, so two operations claiming either collide. + pub(super) fn footprint(&self, footprint: &mut Footprint) { + footprint.add(Coordinate::BaseName(self.base.name.clone())); + footprint.add(Coordinate::BaseLocation(self.base.path.clone())); + } +} + +impl From<&AddBase> for pb::AddBase { + fn from(value: &AddBase) -> Self { + Self { + local: value.local, + base: Some(pb::BasePath::from(value.base.clone())), + } + } +} + +impl TryFrom for AddBase { + type Error = Error; + + fn try_from(message: pb::AddBase) -> Result { + Ok(Self { + local: message.local, + base: BasePath::from(required(message.base, "AddBase.base")?), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::transaction::action::Action; + use crate::transaction::action::test_support::apply; + use crate::transaction::test_support::sample_manifest; + + #[test] + fn test_add_base_mints_an_id_and_rejects_duplicates() { + let manifest = sample_manifest(); + let next = apply( + &manifest, + vec![Action::AddBase(AddBase { + local: 0, + base: BasePath::new(0, "s3://bucket/a".into(), Some("a".into()), false), + })], + ) + .unwrap(); + assert_eq!(next.base_paths.len(), 1); + assert_eq!(next.base_paths[&1].path, "s3://bucket/a"); + + let error = apply( + &next, + vec![Action::AddBase(AddBase { + local: 0, + base: BasePath::new(0, "s3://bucket/a".into(), Some("other".into()), false), + })], + ) + .unwrap_err(); + assert!( + error.to_string().contains("already exists"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_two_add_bases_in_one_operation_see_each_other() { + let manifest = sample_manifest(); + let error = apply( + &manifest, + vec![ + Action::AddBase(AddBase { + local: 0, + base: BasePath::new(0, "s3://bucket/a".into(), Some("a".into()), false), + }), + Action::AddBase(AddBase { + local: 1, + base: BasePath::new(0, "s3://bucket/b".into(), Some("a".into()), false), + }), + ], + ) + .unwrap_err(); + assert!( + error.to_string().contains("already exists"), + "unexpected error: {error}" + ); + } +} diff --git a/rust/lance-table/src/transaction/action/add_data_file.rs b/rust/lance-table/src/transaction/action/add_data_file.rs new file mode 100644 index 00000000000..6cc141f5c7f --- /dev/null +++ b/rust/lance-table/src/transaction/action/add_data_file.rs @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Add a data file to a fragment. + +use super::apply::ApplyState; +use super::proto::{data_change_from_wire, data_change_to_wire, required}; +use super::{Footprint, Ref}; +use crate::format::{DataFile, pb}; +use lance_core::deepsize::DeepSizeOf; +use lance_core::{Error, Result}; + +/// Add a data file to a fragment. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct AddDataFile { + /// The fragment to add the file to: committed, or a fragment minted earlier + /// in the same operation. + pub fragment: Ref, + /// The file. Its `fields` are placeholders and are stamped in at apply from + /// `field_ids`, which is the authority for the column -> field mapping. + pub file: DataFile, + /// One entry per column in `file`, in column order. + pub field_ids: Vec, + /// See [`AddFragment::data_change`](super::AddFragment::data_change). + pub data_change: bool, +} + +impl AddDataFile { + pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { + let fragment_id = state.resolve_fragment(self.fragment)?; + let field_ids = self + .field_ids + .iter() + .map(|field| state.resolve_field(*field)) + .collect::>>()?; + + let mut file = self.file.clone(); + if !file.column_indices.is_empty() && file.column_indices.len() != field_ids.len() { + return Err(Error::invalid_input(format!( + "AddDataFile for fragment {fragment_id} lists {} field ids but the file has {} \ + columns", + field_ids.len(), + file.column_indices.len() + ))); + } + file.fields = field_ids.into(); + + state + .fragment_mut(fragment_id, "AddDataFile")? + .files + .push(file); + Ok(()) + } + + /// The data of every committed field the file backs, in the fragment it is + /// attached to. A file backing only minted fields writes nothing. + pub(super) fn footprint(&self, footprint: &mut Footprint) { + footprint.add_field_data( + self.fragment, + self.field_ids + .iter() + .filter_map(|field| field.committed().and_then(|id| i32::try_from(id).ok())), + ); + } +} + +impl From<&AddDataFile> for pb::AddDataFile { + fn from(value: &AddDataFile) -> Self { + Self { + fragment: Some(value.fragment.into()), + file: Some(pb::DataFile::from(&value.file)), + field_ids: value.field_ids.iter().map(|id| (*id).into()).collect(), + data_change: data_change_to_wire(value.data_change), + } + } +} + +impl TryFrom for AddDataFile { + type Error = Error; + + fn try_from(message: pb::AddDataFile) -> Result { + Ok(Self { + fragment: required(message.fragment, "AddDataFile.fragment")?.try_into()?, + file: DataFile::try_from(required(message.file, "AddDataFile.file")?)?, + field_ids: message + .field_ids + .into_iter() + .map(Ref::try_from) + .collect::>>()?, + data_change: data_change_from_wire(message.data_change), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::transaction::action::Action; + use crate::transaction::action::test_support::apply; + use crate::transaction::test_support::sample_manifest; + + #[test] + fn test_add_data_file_rejects_an_unbound_local_token() { + let manifest = sample_manifest(); + let error = apply( + &manifest, + vec![Action::AddDataFile(AddDataFile { + fragment: Ref::Local(3), + file: DataFile::new_unstarted("data/x.lance", 2, 0), + field_ids: vec![Ref::Committed(0)], + data_change: true, + })], + ) + .unwrap_err(); + assert!( + error.to_string().contains("local fragment token 3"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_add_data_file_rejects_a_column_count_mismatch() { + let manifest = sample_manifest(); + let mut file = DataFile::new_unstarted("data/x.lance", 2, 0); + file.column_indices = vec![0, 1].into(); + + let error = apply( + &manifest, + vec![Action::AddDataFile(AddDataFile { + fragment: Ref::Committed(0), + file, + field_ids: vec![Ref::Committed(0)], + data_change: true, + })], + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + assert!( + error.to_string().contains("1 field ids"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_add_data_file_rejects_a_missing_fragment() { + let manifest = sample_manifest(); + let error = apply( + &manifest, + vec![Action::AddDataFile(AddDataFile { + fragment: Ref::Committed(7), + file: DataFile::new_unstarted("data/x.lance", 2, 0), + field_ids: vec![Ref::Committed(0)], + data_change: true, + })], + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + assert!(error.to_string().contains("fragment 7"), "{error}"); + } +} diff --git a/rust/lance-table/src/transaction/action/add_field.rs b/rust/lance-table/src/transaction/action/add_field.rs new file mode 100644 index 00000000000..a82d210cf21 --- /dev/null +++ b/rust/lance-table/src/transaction/action/add_field.rs @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Mint a new schema field. + +use super::apply::ApplyState; +use super::proto::required; +use super::{Footprint, Ref}; +use crate::format::pb; +use lance_core::datatypes::Field; +use lance_core::deepsize::DeepSizeOf; +use lance_core::{Error, Result}; + +/// Mint a new schema field. +/// +/// A nested column that introduces several fields is several ordered +/// `AddField`s -- parent first, each child naming its parent's local token. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct AddField { + /// Token standing in for the field id until it is allocated at apply. + pub local: u32, + /// The parent field, or `None` for a top-level column. + pub parent: Option, + /// The field definition. Its `id`, `parent_id`, and `children` are ignored: + /// `local` and `parent` carry that structure, and each child is its own + /// action. + pub def: Field, +} + +impl AddField { + pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { + let id = state.mint_field(self.local)?; + let parent_id = self + .parent + .map(|parent| state.resolve_field(parent)) + .transpose()?; + + // The definition's own id, parent id, and children are ignored: the + // minted id and the parent reference carry that structure, and each + // child column arrives as its own action. + let field = Field { + id, + parent_id: parent_id.unwrap_or(-1), + children: Vec::new(), + ..self.def.clone() + }; + + match parent_id { + None => state.schema_mut().fields.push(field), + Some(parent_id) => { + let parent = state + .schema_mut() + .field_by_id_mut(parent_id) + .ok_or_else(|| { + Error::invalid_input(format!( + "AddField names parent field {parent_id}, which does not exist" + )) + })?; + parent.children.push(field); + } + } + Ok(()) + } + + /// Nothing: the field does not exist in the read version. Attaching it + /// under a committed parent does not rewrite the parent's definition. + pub(super) fn footprint(&self, _footprint: &mut Footprint) {} +} + +impl From<&AddField> for pb::AddField { + fn from(value: &AddField) -> Self { + Self { + local: value.local, + parent: value.parent.map(Into::into), + def: Some(lance_file::format::pb::Field::from(&value.def)), + } + } +} + +impl TryFrom for AddField { + type Error = Error; + + fn try_from(message: pb::AddField) -> Result { + Ok(Self { + local: message.local, + parent: message.parent.map(Ref::try_from).transpose()?, + def: Field::from(&required(message.def, "AddField.def")?), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::DataFile; + use crate::transaction::action::test_support::{added_field, apply}; + use crate::transaction::action::{Action, AddDataFile}; + use crate::transaction::test_support::sample_manifest; + use arrow_schema::{DataType, Field as ArrowField}; + + #[test] + fn test_two_add_fields_mint_distinct_ids() { + let manifest = sample_manifest(); + let next = apply( + &manifest, + vec![ + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("a"), + }), + Action::AddField(AddField { + local: 1, + parent: None, + def: added_field("b"), + }), + ], + ) + .unwrap(); + + let ids = next + .schema + .fields + .iter() + .map(|f| (f.name.as_str(), f.id)) + .collect::>(); + assert_eq!(ids, vec![("id", 0), ("a", 1), ("b", 2)]); + } + + #[test] + fn test_add_field_then_add_its_data_file() { + // The add-column shape: mint the field, then write the file that backs + // it, naming the field by the token the mint has not resolved yet. + let manifest = sample_manifest(); + let next = apply( + &manifest, + vec![ + Action::AddField(AddField { + local: 7, + parent: None, + def: added_field("added"), + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Committed(0), + file: DataFile::new_unstarted("data/added.lance", 2, 0), + field_ids: vec![Ref::Local(7)], + data_change: true, + }), + ], + ) + .unwrap(); + + let field_id = next.schema.field("added").unwrap().id; + assert_eq!(field_id, 1); + let fragment = next.fragments.iter().find(|f| f.id == 0).unwrap(); + assert_eq!(fragment.files.last().unwrap().fields.as_ref(), &[field_id]); + } + + #[test] + fn test_add_field_under_a_parent() { + let manifest = sample_manifest(); + let next = apply( + &manifest, + vec![ + Action::AddField(AddField { + local: 0, + parent: None, + def: Field::try_from(ArrowField::new( + "nested", + DataType::Struct(Default::default()), + true, + )) + .unwrap(), + }), + Action::AddField(AddField { + local: 1, + parent: Some(Ref::Local(0)), + def: added_field("child"), + }), + ], + ) + .unwrap(); + + let parent = next.schema.field("nested").unwrap(); + assert_eq!(parent.children.len(), 1); + assert_eq!(parent.children[0].name, "child"); + assert_eq!(parent.children[0].parent_id, parent.id); + } + + #[test] + fn test_add_field_rejects_a_missing_parent() { + let manifest = sample_manifest(); + let error = apply( + &manifest, + vec![Action::AddField(AddField { + local: 0, + parent: Some(Ref::Committed(7)), + def: added_field("orphan"), + })], + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + assert!(error.to_string().contains("parent field 7"), "{error}"); + } + + #[test] + fn test_duplicate_local_token_is_rejected() { + let manifest = sample_manifest(); + let error = apply( + &manifest, + vec![ + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("a"), + }), + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("b"), + }), + ], + ) + .unwrap_err(); + assert!( + error.to_string().contains("minted more than once"), + "unexpected error: {error}" + ); + } +} diff --git a/rust/lance-table/src/transaction/action/add_fragment.rs b/rust/lance-table/src/transaction/action/add_fragment.rs new file mode 100644 index 00000000000..e172eaa57de --- /dev/null +++ b/rust/lance-table/src/transaction/action/add_fragment.rs @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Mint a new, empty fragment. + +use super::Footprint; +use super::apply::ApplyState; +use super::proto::{data_change_from_wire, data_change_to_wire}; +use crate::format::{ExternalFile, Fragment, RowIdMeta, pb}; +use crate::rowids::version::RowDatasetVersionMeta; +use lance_core::Result; +use lance_core::deepsize::DeepSizeOf; + +/// Mint a new, empty fragment. +/// +/// Its data files arrive via [`AddDataFile`](super::AddDataFile) actions naming +/// this fragment's local token. A freshly-minted fragment has no deletion +/// vector: it has no committed rows to delete yet. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct AddFragment { + /// Token standing in for the fragment id until it is allocated at apply. + pub local: u32, + /// Physical rows in the fragment, including rows later tombstoned. + pub physical_rows: u64, + /// Stable row id sequence. `None` on datasets without stable row ids, and + /// on datasets that have them but where the ids are assigned at apply. + pub row_id_meta: Option, + /// Per-row version metadata, carried exactly as on + /// [`Fragment`](crate::format::Fragment). `None` means "stamp at apply". + pub last_updated_at_version_meta: Option, + pub created_at_version_meta: Option, + /// `false` marks a pure rearrangement, e.g. a compaction rewrite. + pub data_change: bool, +} + +impl AddFragment { + pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { + let id = state.mint_fragment(self.local)?; + state.push_fragment(Fragment { + id, + files: Vec::new(), + overlays: Vec::new(), + deletion_file: None, + row_id_meta: self.row_id_meta.clone(), + physical_rows: Some(self.physical_rows as usize), + last_updated_at_version_meta: self.last_updated_at_version_meta.clone(), + created_at_version_meta: self.created_at_version_meta.clone(), + }); + Ok(()) + } + + /// Nothing: the fragment does not exist in the read version, so no + /// concurrent writer can be naming it. + pub(super) fn footprint(&self, _footprint: &mut Footprint) {} +} + +fn external_file_to_wire(file: &ExternalFile) -> pb::ExternalFile { + pb::ExternalFile { + path: file.path.clone(), + offset: file.offset, + size: file.size, + } +} + +fn external_file_from_wire(file: pb::ExternalFile) -> ExternalFile { + ExternalFile { + path: file.path, + offset: file.offset, + size: file.size, + } +} + +impl From<&AddFragment> for pb::AddFragment { + fn from(value: &AddFragment) -> Self { + Self { + local: value.local, + physical_rows: value.physical_rows, + row_id_sequence: value.row_id_meta.as_ref().map(|meta| match meta { + RowIdMeta::Inline(data) => { + pb::add_fragment::RowIdSequence::InlineRowIds(data.clone()) + } + RowIdMeta::External(file) => { + pb::add_fragment::RowIdSequence::ExternalRowIds(external_file_to_wire(file)) + } + }), + last_updated_at_version_sequence: value + .last_updated_at_version_meta + .as_ref() + .map(|meta| match meta { + RowDatasetVersionMeta::Inline(data) => { + pb::add_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions( + data.to_vec(), + ) + } + RowDatasetVersionMeta::External(file) => { + pb::add_fragment::LastUpdatedAtVersionSequence::ExternalLastUpdatedAtVersions( + external_file_to_wire(file), + ) + } + }), + created_at_version_sequence: value + .created_at_version_meta + .as_ref() + .map(|meta| match meta { + RowDatasetVersionMeta::Inline(data) => { + pb::add_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions( + data.to_vec(), + ) + } + RowDatasetVersionMeta::External(file) => { + pb::add_fragment::CreatedAtVersionSequence::ExternalCreatedAtVersions( + external_file_to_wire(file), + ) + } + }), + data_change: data_change_to_wire(value.data_change), + } + } +} + +impl TryFrom for AddFragment { + type Error = lance_core::Error; + + fn try_from(message: pb::AddFragment) -> Result { + Ok(Self { + local: message.local, + physical_rows: message.physical_rows, + row_id_meta: message.row_id_sequence.map(|sequence| match sequence { + pb::add_fragment::RowIdSequence::InlineRowIds(data) => RowIdMeta::Inline(data), + pb::add_fragment::RowIdSequence::ExternalRowIds(file) => { + RowIdMeta::External(external_file_from_wire(file)) + } + }), + last_updated_at_version_meta: message.last_updated_at_version_sequence.map( + |sequence| { + match sequence { + pb::add_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions( + data, + ) => RowDatasetVersionMeta::Inline(data.into()), + pb::add_fragment::LastUpdatedAtVersionSequence::ExternalLastUpdatedAtVersions( + file, + ) => RowDatasetVersionMeta::External(external_file_from_wire(file)), + } + }, + ), + created_at_version_meta: message.created_at_version_sequence.map(|sequence| { + match sequence { + pb::add_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions(data) => { + RowDatasetVersionMeta::Inline(data.into()) + } + pb::add_fragment::CreatedAtVersionSequence::ExternalCreatedAtVersions(file) => { + RowDatasetVersionMeta::External(external_file_from_wire(file)) + } + } + }), + data_change: data_change_from_wire(message.data_change), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::DataFile; + use crate::transaction::action::test_support::apply; + use crate::transaction::action::{Action, AddDataFile, Ref}; + use crate::transaction::test_support::sample_manifest; + + #[test] + fn test_add_fragment_and_data_file_mint_ids() { + let manifest = sample_manifest(); + let next = apply( + &manifest, + vec![ + Action::AddFragment(AddFragment { + local: 0, + physical_rows: 10, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Local(0), + file: DataFile::new_unstarted("data/new.lance", 2, 0), + field_ids: vec![Ref::Committed(0)], + data_change: true, + }), + ], + ) + .unwrap(); + + // sample_manifest already holds fragment 0, so the mint lands on 1. + assert_eq!( + next.fragments.iter().map(|f| f.id).collect::>(), + vec![0, 1] + ); + let minted = next.fragments.iter().find(|f| f.id == 1).unwrap(); + assert_eq!(minted.physical_rows, Some(10)); + assert_eq!(minted.files.len(), 1); + // The file's field list is stamped in from the action's refs. + assert_eq!(minted.files[0].fields.as_ref(), &[0]); + assert_eq!(next.max_fragment_id(), Some(1)); + } +} diff --git a/rust/lance-table/src/transaction/action/alter_field.rs b/rust/lance-table/src/transaction/action/alter_field.rs new file mode 100644 index 00000000000..545d07e7d87 --- /dev/null +++ b/rust/lance-table/src/transaction/action/alter_field.rs @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Alter facets of an existing field in place. + +use super::apply::ApplyState; +use super::proto::field_id_from_wire; +use super::{Coordinate, Footprint}; +use crate::format::pb; +use lance_core::deepsize::DeepSizeOf; +use lance_core::{Error, Result}; + +/// Alter facets of an existing field in place, preserving its id. +/// +/// Each facet is independently optional -- present means "change this", absent +/// means "leave it alone" -- so a widening cast and a nullability relaxation on +/// the same field commute. A cast additionally needs a +/// [`TombstoneFieldData`](super::TombstoneFieldData) plus a fresh +/// [`AddDataFile`](super::AddDataFile) to rewrite the data. +#[derive(Debug, Clone, PartialEq, DeepSizeOf, Default)] +pub struct AlterField { + pub field: i32, + pub name: Option, + /// The new Arrow logical type. The cast. + pub logical_type: Option, + pub nullable: Option, +} + +impl AlterField { + pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { + let field = state + .schema_mut() + .field_by_id_mut(self.field) + .ok_or_else(|| { + Error::invalid_input(format!( + "AlterField names field {}, which does not exist", + self.field + )) + })?; + if let Some(name) = &self.name { + field.name.clone_from(name); + } + if let Some(nullable) = self.nullable { + field.nullable = nullable; + } + if let Some(logical_type) = &self.logical_type { + field.logical_type = logical_type.as_str().into(); + // The cast leaves any index on the field describing the old type. + // The data rewrite itself is separate actions; this only records + // that every fragment's view of the field changed. + state.rebind_field_everywhere(self.field); + } + Ok(()) + } + + /// The field's definition. The data rewrite a cast needs is separate + /// actions, which record their own coordinates. + pub(super) fn footprint(&self, footprint: &mut Footprint) { + footprint.add(Coordinate::FieldDefinition(self.field)); + } +} + +impl From<&AlterField> for pb::AlterField { + fn from(value: &AlterField) -> Self { + Self { + field: value.field as u64, + name: value.name.clone(), + logical_type: value.logical_type.clone(), + nullable: value.nullable, + } + } +} + +impl TryFrom for AlterField { + type Error = Error; + + fn try_from(message: pb::AlterField) -> Result { + Ok(Self { + field: field_id_from_wire(message.field)?, + name: message.name, + logical_type: message.logical_type, + nullable: message.nullable, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::transaction::action::Action; + use crate::transaction::action::test_support::{apply, apply_with_indices, backed_manifest}; + use crate::transaction::test_support::sample_index_metadata; + + #[test] + fn test_alter_field_renames_without_touching_indices() { + let (next, indices) = apply_with_indices( + &backed_manifest(), + vec![Action::AlterField(AlterField { + field: 0, + name: Some("renamed".into()), + logical_type: None, + nullable: Some(true), + })], + vec![sample_index_metadata("idx")], + ) + .unwrap(); + + let field = next.schema.field_by_id(0).unwrap(); + assert_eq!(field.name, "renamed"); + assert!(field.nullable); + // A rename does not change the values the index recorded. + assert!(indices[0].fragment_bitmap.as_ref().unwrap().contains(0)); + } + + #[test] + fn test_alter_field_retype_prunes_covering_indices() { + let (next, indices) = apply_with_indices( + &backed_manifest(), + vec![Action::AlterField(AlterField { + field: 0, + name: None, + logical_type: Some("int64".into()), + nullable: None, + })], + vec![sample_index_metadata("idx")], + ) + .unwrap(); + + assert_eq!( + next.schema.field_by_id(0).unwrap().logical_type.to_string(), + "int64" + ); + assert!(indices[0].fragment_bitmap.as_ref().unwrap().is_empty()); + } + + #[test] + fn test_alter_field_rejects_a_missing_field() { + let error = apply( + &backed_manifest(), + vec![Action::AlterField(AlterField { + field: 7, + name: Some("nope".into()), + ..Default::default() + })], + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + assert!(error.to_string().contains("field 7"), "{error}"); + } +} diff --git a/rust/lance-table/src/transaction/action/apply.rs b/rust/lance-table/src/transaction/action/apply.rs index fa1cee8a135..719b47be2b4 100644 --- a/rust/lance-table/src/transaction/action/apply.rs +++ b/rust/lance-table/src/transaction/action/apply.rs @@ -9,21 +9,22 @@ //! that token -- in this same operation -- resolves to the id this apply chose. //! Replaying the same action set against a different version therefore produces //! different ids without any of the actions changing. +//! +//! [`ApplyState`] is that working copy, and its methods are the API the action +//! modules program against. What each action does with it lives in that action's +//! own module. -use super::{ - Action, AddBase, AddDataFile, AddField, AddFragment, AlterField, DropField, Ref, - RemoveFragment, SetDeletionFile, TombstoneFieldData, UserOperation, -}; +use super::{Ref, UserOperation}; use crate::format::{BasePath, Fragment, IndexMetadata, Manifest, ManifestBuildConfig}; use crate::rowids::version::build_version_meta; use crate::transaction::Transaction; -use lance_core::datatypes::{Field, Schema}; +use lance_core::datatypes::Schema; use lance_core::{Error, Result}; use std::collections::{HashMap, HashSet}; /// The field id written into a data file's field list once the file no longer /// backs that field. A file whose every slot is tombstoned is dropped. -const TOMBSTONED_FIELD: i32 = -2; +pub(super) const TOMBSTONED_FIELD: i32 = -2; impl Transaction { /// Build the next manifest by applying an action set. @@ -54,7 +55,7 @@ impl Transaction { let new_version = current_manifest.version + 1; let mut state = ApplyState::new(current_manifest); for action in user_operation.iter_actions() { - state.apply(action)?; + action.apply(&mut state)?; } let mut next_row_id = current_manifest @@ -99,7 +100,7 @@ impl Transaction { /// The read-version state an action set is applied against, plus the id /// allocations made so far. -struct ApplyState { +pub(super) struct ApplyState { schema: Schema, fragments: Vec, /// Base paths minted by this operation. Kept apart from the manifest's own @@ -147,271 +148,140 @@ impl ApplyState { } } - fn apply(&mut self, action: &Action) -> Result<()> { - match action { - Action::AddFragment(action) => self.add_fragment(action), - Action::AddDataFile(action) => self.add_data_file(action), - Action::AddField(action) => self.add_field(action), - Action::AddBase(action) => self.add_base(action), - Action::TombstoneFieldData(action) => self.tombstone_field_data(action), - Action::RemoveFragment(action) => self.remove_fragment(action), - Action::SetDeletionFile(action) => self.set_deletion_file(action), - Action::AlterField(action) => self.alter_field(action), - Action::DropField(action) => self.drop_field(action), - } + pub(super) fn schema(&self) -> &Schema { + &self.schema } - fn add_fragment(&mut self, action: &AddFragment) -> Result<()> { - if self.fragment_tokens.contains_key(&action.local) { - return Err(duplicate_token_err("fragment", action.local)); - } - let id = self.next_fragment_id; - self.next_fragment_id += 1; - self.fragment_tokens.insert(action.local, id); - self.minted_fragments.insert(id); - - self.fragments.push(Fragment { - id, - files: Vec::new(), - overlays: Vec::new(), - deletion_file: None, - row_id_meta: action.row_id_meta.clone(), - physical_rows: Some(action.physical_rows as usize), - last_updated_at_version_meta: action.last_updated_at_version_meta.clone(), - created_at_version_meta: action.created_at_version_meta.clone(), - }); - Ok(()) + pub(super) fn schema_mut(&mut self) -> &mut Schema { + &mut self.schema } - fn add_data_file(&mut self, action: &AddDataFile) -> Result<()> { - let fragment_id = self.resolve_fragment(action.fragment)?; - let field_ids = action - .field_ids - .iter() - .map(|field| self.resolve_field(*field)) - .collect::>>()?; - - let mut file = action.file.clone(); - if !file.column_indices.is_empty() && file.column_indices.len() != field_ids.len() { - return Err(Error::invalid_input(format!( - "AddDataFile for fragment {fragment_id} lists {} field ids but the file has {} \ - columns", - field_ids.len(), - file.column_indices.len() - ))); - } - file.fields = field_ids.into(); + pub(super) fn fragments_mut(&mut self) -> &mut [Fragment] { + &mut self.fragments + } - let fragment = self - .fragments + pub(super) fn fragment_mut(&mut self, fragment_id: u64, action: &str) -> Result<&mut Fragment> { + self.fragments .iter_mut() .find(|fragment| fragment.id == fragment_id) .ok_or_else(|| { Error::invalid_input(format!( - "AddDataFile targets fragment {fragment_id}, which does not exist" + "{action} targets fragment {fragment_id}, which does not exist" )) - })?; - fragment.files.push(file); - Ok(()) + }) } - fn add_field(&mut self, action: &AddField) -> Result<()> { - let id = self.next_field_id; - self.next_field_id += 1; - if self.field_tokens.contains_key(&action.local) { - return Err(duplicate_token_err("field", action.local)); - } - self.field_tokens.insert(action.local, id); - - let parent_id = action - .parent - .map(|parent| self.resolve_field(parent)) - .transpose()?; - - // The definition's own id, parent id, and children are ignored: the - // minted id and the parent reference carry that structure, and each - // child column arrives as its own action. - let field = Field { - id, - parent_id: parent_id.unwrap_or(-1), - children: Vec::new(), - ..action.def.clone() - }; - - match parent_id { - None => self.schema.fields.push(field), - Some(parent_id) => { - let parent = self.schema.field_by_id_mut(parent_id).ok_or_else(|| { - Error::invalid_input(format!( - "AddField names parent field {parent_id}, which does not exist" - )) - })?; - parent.children.push(field); - } - } - Ok(()) + pub(super) fn push_fragment(&mut self, fragment: Fragment) { + self.fragments.push(fragment); } - fn add_base(&mut self, action: &AddBase) -> Result<()> { - let id = self.next_base_id; - self.next_base_id += 1; - if self.base_tokens.contains_key(&action.local) { - return Err(duplicate_token_err("base", action.local)); + /// Drop a fragment and forget everything recorded about it. `false` if no + /// such fragment was present. + pub(super) fn remove_fragment(&mut self, fragment_id: u64) -> bool { + let before = self.fragments.len(); + self.fragments.retain(|fragment| fragment.id != fragment_id); + if self.fragments.len() == before { + return false; } - self.base_tokens.insert(action.local, id); + self.minted_fragments.remove(&fragment_id); + self.rebound_fields.remove(&fragment_id); + true + } - let conflicting = self - .existing_base_paths + /// The base paths this apply can see: the read version's, plus the ones + /// earlier actions in this operation minted. + pub(super) fn bases(&self) -> impl Iterator { + self.existing_base_paths .values() .chain(self.new_bases.iter()) - .find(|base| base.name == action.base.name || base.path == action.base.path); - if let Some(conflicting) = conflicting { - return Err(Error::invalid_input(format!( - "Conflict detected: Base path with name '{:?}' or path '{}' already exists. \ - Existing: name='{:?}', path='{}'", - action.base.name, action.base.path, conflicting.name, conflicting.path - ))); - } + } - let mut base = action.base.clone(); - base.id = id; + pub(super) fn push_base(&mut self, base: BasePath) { self.new_bases.push(base); - Ok(()) } - fn tombstone_field_data(&mut self, action: &TombstoneFieldData) -> Result<()> { - let fragment_id = self.resolve_fragment(action.fragment)?; - let fragment = fragment_mut(&mut self.fragments, fragment_id, "TombstoneFieldData")?; - - for &field_id in &action.field_ids { - let mut found = false; - for file in fragment.files.iter_mut() { - let Some(position) = file.fields.iter().position(|id| *id == field_id) else { - continue; - }; - let mut fields = file.fields.to_vec(); - fields[position] = TOMBSTONED_FIELD; - file.fields = fields.into(); - found = true; - } - if !found { - return Err(Error::invalid_input(format!( - "TombstoneFieldData names field {field_id}, which no data file in fragment \ - {fragment_id} backs" - ))); - } + pub(super) fn mint_fragment(&mut self, token: u32) -> Result { + if self.fragment_tokens.contains_key(&token) { + return Err(duplicate_token_err("fragment", token)); } - - // New values for these fields supersede any overlay still shadowing - // them, so the drop is not silently masked by stale overlay cells. - let overlaid: Vec = action - .field_ids - .iter() - .filter_map(|id| u32::try_from(*id).ok()) - .collect(); - crate::format::overlay::tombstone_overlay_fields(&mut fragment.overlays, &overlaid); - - self.rebound_fields - .entry(fragment_id) - .or_default() - .extend(action.field_ids.iter().copied()); - Ok(()) + let id = self.next_fragment_id; + self.next_fragment_id += 1; + self.fragment_tokens.insert(token, id); + self.minted_fragments.insert(id); + Ok(id) } - fn remove_fragment(&mut self, action: &RemoveFragment) -> Result<()> { - let fragment_id = self.resolve_fragment(action.fragment)?; - let before = self.fragments.len(); - self.fragments.retain(|fragment| fragment.id != fragment_id); - if self.fragments.len() == before { - return Err(Error::invalid_input(format!( - "RemoveFragment targets fragment {fragment_id}, which does not exist" - ))); + pub(super) fn mint_field(&mut self, token: u32) -> Result { + if self.field_tokens.contains_key(&token) { + return Err(duplicate_token_err("field", token)); } - self.minted_fragments.remove(&fragment_id); - self.rebound_fields.remove(&fragment_id); - Ok(()) + let id = self.next_field_id; + self.next_field_id += 1; + self.field_tokens.insert(token, id); + Ok(id) } - fn set_deletion_file(&mut self, action: &SetDeletionFile) -> Result<()> { - let fragment = fragment_mut(&mut self.fragments, action.fragment, "SetDeletionFile")?; - fragment.deletion_file = action.deletion_file.clone(); - Ok(()) + pub(super) fn mint_base(&mut self, token: u32) -> Result { + if self.base_tokens.contains_key(&token) { + return Err(duplicate_token_err("base", token)); + } + let id = self.next_base_id; + self.next_base_id += 1; + self.base_tokens.insert(token, id); + Ok(id) } - fn alter_field(&mut self, action: &AlterField) -> Result<()> { - let field = self.schema.field_by_id_mut(action.field).ok_or_else(|| { - Error::invalid_input(format!( - "AlterField names field {}, which does not exist", - action.field - )) - })?; - if let Some(name) = &action.name { - field.name.clone_from(name); - } - if let Some(nullable) = action.nullable { - field.nullable = nullable; - } - if let Some(logical_type) = &action.logical_type { - field.logical_type = logical_type.as_str().into(); - // The cast leaves any index on the field describing the old type. - // The data rewrite itself is separate actions; this only records - // that every fragment's view of the field changed. - for fragment in &self.fragments { - self.rebound_fields - .entry(fragment.id) - .or_default() - .insert(action.field); - } + pub(super) fn resolve_fragment(&self, reference: Ref) -> Result { + match reference { + Ref::Committed(id) => Ok(id), + Ref::Local(token) => self + .fragment_tokens + .get(&token) + .copied() + .ok_or_else(|| unbound_token_err("fragment", token)), } - Ok(()) } - fn drop_field(&mut self, action: &DropField) -> Result<()> { - let field = self.schema.field_by_id(action.field).ok_or_else(|| { - Error::invalid_input(format!( - "DropField names field {}, which does not exist", - action.field - )) - })?; - // A struct's children cannot outlive it, so the whole subtree goes. - let mut dropped = HashSet::new(); - collect_subtree_ids(field, &mut dropped); - remove_field(&mut self.schema.fields, action.field); + pub(super) fn resolve_field(&self, reference: Ref) -> Result { + match reference { + Ref::Committed(id) => i32::try_from(id).map_err(|_| { + Error::invalid_input(format!("field id {id} in an action is out of range")) + }), + Ref::Local(token) => self + .field_tokens + .get(&token) + .copied() + .ok_or_else(|| unbound_token_err("field", token)), + } + } - // The fields are gone from the schema, so the slots that backed them in - // each data file are dead. Tombstoning rather than rewriting the field - // list keeps a file's remaining columns at the positions they were - // written at; a file left with nothing live is pruned during - // normalization. - for fragment in self.fragments.iter_mut() { - for file in fragment.files.iter_mut() { - if !file.fields.iter().any(|id| dropped.contains(id)) { - continue; - } - let fields = file - .fields - .iter() - .map(|id| { - if dropped.contains(id) { - TOMBSTONED_FIELD - } else { - *id - } - }) - .collect::>(); - file.fields = fields.into(); - } + /// Record that these fields' data in this fragment no longer matches what + /// an index built over them recorded. + pub(super) fn rebind_fields( + &mut self, + fragment_id: u64, + fields: impl IntoIterator, + ) { + self.rebound_fields + .entry(fragment_id) + .or_default() + .extend(fields); + } - let overlaid: Vec = dropped - .iter() - .filter_map(|id| u32::try_from(*id).ok()) - .collect(); - crate::format::overlay::tombstone_overlay_fields(&mut fragment.overlays, &overlaid); + /// As [`Self::rebind_fields`], for a change that invalidates the field + /// across every fragment at once. + pub(super) fn rebind_field_everywhere(&mut self, field: i32) { + let fragment_ids = self + .fragments + .iter() + .map(|fragment| fragment.id) + .collect::>(); + for fragment_id in fragment_ids { + self.rebound_fields + .entry(fragment_id) + .or_default() + .insert(field); } - - // Indices over a field that no longer exists are discarded wholesale by - // `retain_relevant_indices`, so there is nothing to record here. - Ok(()) } /// Stamp row ids and version metadata onto the fragments this operation @@ -450,64 +320,6 @@ impl ApplyState { self.minted_fragments = minted_ids; Ok(()) } - - fn resolve_fragment(&self, reference: Ref) -> Result { - match reference { - Ref::Committed(id) => Ok(id), - Ref::Local(token) => self - .fragment_tokens - .get(&token) - .copied() - .ok_or_else(|| unbound_token_err("fragment", token)), - } - } - - fn resolve_field(&self, reference: Ref) -> Result { - match reference { - Ref::Committed(id) => i32::try_from(id).map_err(|_| { - Error::invalid_input(format!("field id {id} in an action is out of range")) - }), - Ref::Local(token) => self - .field_tokens - .get(&token) - .copied() - .ok_or_else(|| unbound_token_err("field", token)), - } - } -} - -fn collect_subtree_ids(field: &Field, out: &mut HashSet) { - out.insert(field.id); - for child in &field.children { - collect_subtree_ids(child, out); - } -} - -/// Remove the field with `field_id` from `fields`, at whatever depth it sits. -fn remove_field(fields: &mut Vec, field_id: i32) { - let before = fields.len(); - fields.retain(|field| field.id != field_id); - if fields.len() != before { - return; - } - for field in fields.iter_mut() { - remove_field(&mut field.children, field_id); - } -} - -fn fragment_mut<'a>( - fragments: &'a mut [Fragment], - fragment_id: u64, - action: &str, -) -> Result<&'a mut Fragment> { - fragments - .iter_mut() - .find(|fragment| fragment.id == fragment_id) - .ok_or_else(|| { - Error::invalid_input(format!( - "{action} targets fragment {fragment_id}, which does not exist" - )) - }) } /// Drop the fragments whose data no longer matches what an index recorded. @@ -554,593 +366,11 @@ fn duplicate_token_err(space: &str, token: u32) -> Error { #[cfg(test)] mod tests { use super::*; - use crate::format::{DataFile, DeletionFile, DeletionFileType}; + use crate::format::DataFile; use crate::transaction::Operation; - use crate::transaction::action::UserAction; - use crate::transaction::test_support::{ - default_build_config, sample_index_metadata, sample_manifest, - }; - use arrow_schema::{DataType, Field as ArrowField}; - use std::sync::Arc; - - fn apply(manifest: &Manifest, actions: Vec) -> Result { - apply_with_indices(manifest, actions, Vec::new()).map(|(manifest, _)| manifest) - } - - fn apply_with_indices( - manifest: &Manifest, - actions: Vec, - indices: Vec, - ) -> Result<(Manifest, Vec)> { - let transaction = Transaction::new( - manifest.version, - Operation::UserOperation(UserOperation::new( - "test", - vec![UserAction::new("step", actions)], - )), - None, - ); - transaction.build_manifest(Some(manifest), indices, "tx.txn", &default_build_config()) - } - - fn added_field(name: &str) -> Field { - Field::try_from(ArrowField::new(name, DataType::Int32, true)).unwrap() - } - - /// `sample_manifest` with fragment 0 actually backed by a data file, so the - /// reference-stable actions have something committed to point at. - fn backed_manifest() -> Manifest { - let mut manifest = sample_manifest(); - let mut fragment = Fragment::new(0); - fragment.physical_rows = Some(10); - fragment.files.push(DataFile::new( - "data/0.lance", - vec![0], - vec![0], - 2, - 0, - None, - None, - )); - manifest.fragments = Arc::new(vec![fragment]); - manifest - } - - #[test] - fn test_tombstone_field_data_drops_the_backing_file() { - let manifest = backed_manifest(); - let next = apply( - &manifest, - vec![Action::TombstoneFieldData(TombstoneFieldData { - fragment: Ref::Committed(0), - field_ids: vec![0], - data_change: true, - })], - ) - .unwrap(); - - // The file backed only field 0, so tombstoning it leaves nothing behind. - assert!(next.fragments[0].files.is_empty()); - } - - #[test] - fn test_tombstone_field_data_keeps_a_file_with_a_live_field() { - let mut manifest = backed_manifest(); - let mut fragment = manifest.fragments[0].clone(); - fragment.files[0] = DataFile::new("data/0.lance", vec![0, 1], vec![0, 1], 2, 0, None, None); - manifest.fragments = Arc::new(vec![fragment]); - - let next = apply( - &manifest, - vec![Action::TombstoneFieldData(TombstoneFieldData { - fragment: Ref::Committed(0), - field_ids: vec![0], - data_change: true, - })], - ) - .unwrap(); - - assert_eq!( - next.fragments[0].files[0].fields.as_ref(), - &[TOMBSTONED_FIELD, 1] - ); - } - - #[test] - fn test_tombstone_field_data_prunes_the_fragment_from_covering_indices() { - let manifest = backed_manifest(); - let (_, indices) = apply_with_indices( - &manifest, - vec![Action::TombstoneFieldData(TombstoneFieldData { - fragment: Ref::Committed(0), - field_ids: vec![0], - data_change: true, - })], - vec![sample_index_metadata("idx")], - ) - .unwrap(); - - // The index covers field 0, whose data in fragment 0 is now gone. - assert!(indices[0].fragment_bitmap.as_ref().unwrap().is_empty()); - } - - #[test] - fn test_tombstone_field_data_rejects_a_field_no_file_backs() { - let manifest = backed_manifest(); - let error = apply( - &manifest, - vec![Action::TombstoneFieldData(TombstoneFieldData { - fragment: Ref::Committed(0), - field_ids: vec![7], - data_change: true, - })], - ) - .unwrap_err(); - - assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); - assert!(error.to_string().contains("field 7"), "{error}"); - } - - #[test] - fn test_remove_fragment() { - let manifest = backed_manifest(); - let next = apply( - &manifest, - vec![Action::RemoveFragment(RemoveFragment { - fragment: Ref::Committed(0), - data_change: true, - })], - ) - .unwrap(); - - assert!(next.fragments.is_empty()); - } - - #[test] - fn test_remove_fragment_can_drop_one_minted_in_the_same_operation() { - let manifest = backed_manifest(); - let next = apply( - &manifest, - vec![ - Action::AddFragment(AddFragment { - local: 0, - physical_rows: 10, - row_id_meta: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - data_change: true, - }), - Action::RemoveFragment(RemoveFragment { - fragment: Ref::Local(0), - data_change: true, - }), - ], - ) - .unwrap(); - - assert_eq!( - next.fragments.iter().map(|f| f.id).collect::>(), - vec![0] - ); - } - - #[test] - fn test_remove_fragment_rejects_a_missing_fragment() { - let manifest = backed_manifest(); - let error = apply( - &manifest, - vec![Action::RemoveFragment(RemoveFragment { - fragment: Ref::Committed(7), - data_change: true, - })], - ) - .unwrap_err(); - - assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); - assert!(error.to_string().contains("fragment 7"), "{error}"); - } - - #[test] - fn test_set_deletion_file_sets_and_clears() { - let manifest = backed_manifest(); - let deletion_file = DeletionFile { - read_version: manifest.version, - id: 3, - file_type: DeletionFileType::Array, - num_deleted_rows: Some(2), - base_id: None, - }; - let next = apply( - &manifest, - vec![Action::SetDeletionFile(SetDeletionFile { - fragment: 0, - deletion_file: Some(deletion_file.clone()), - data_change: true, - })], - ) - .unwrap(); - assert_eq!(next.fragments[0].deletion_file, Some(deletion_file)); - - // An absent deletion file is a request to clear it, not a no-op. - let cleared = apply( - &next, - vec![Action::SetDeletionFile(SetDeletionFile { - fragment: 0, - deletion_file: None, - data_change: true, - })], - ) - .unwrap(); - assert_eq!(cleared.fragments[0].deletion_file, None); - } - - #[test] - fn test_set_deletion_file_rejects_a_missing_fragment() { - let manifest = backed_manifest(); - let error = apply( - &manifest, - vec![Action::SetDeletionFile(SetDeletionFile { - fragment: 7, - deletion_file: None, - data_change: true, - })], - ) - .unwrap_err(); - - assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); - } - - #[test] - fn test_alter_field_renames_without_touching_indices() { - let manifest = backed_manifest(); - let (next, indices) = apply_with_indices( - &manifest, - vec![Action::AlterField(AlterField { - field: 0, - name: Some("renamed".into()), - logical_type: None, - nullable: Some(true), - })], - vec![sample_index_metadata("idx")], - ) - .unwrap(); - - let field = next.schema.field_by_id(0).unwrap(); - assert_eq!(field.name, "renamed"); - assert!(field.nullable); - // A rename does not change the values the index recorded. - assert!(indices[0].fragment_bitmap.as_ref().unwrap().contains(0)); - } - - #[test] - fn test_alter_field_retype_prunes_covering_indices() { - let manifest = backed_manifest(); - let (next, indices) = apply_with_indices( - &manifest, - vec![Action::AlterField(AlterField { - field: 0, - name: None, - logical_type: Some("int64".into()), - nullable: None, - })], - vec![sample_index_metadata("idx")], - ) - .unwrap(); - - assert_eq!( - next.schema.field_by_id(0).unwrap().logical_type.to_string(), - "int64" - ); - assert!(indices[0].fragment_bitmap.as_ref().unwrap().is_empty()); - } - - #[test] - fn test_alter_field_rejects_a_missing_field() { - let manifest = backed_manifest(); - let error = apply( - &manifest, - vec![Action::AlterField(AlterField { - field: 7, - name: Some("nope".into()), - ..Default::default() - })], - ) - .unwrap_err(); - - assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); - assert!(error.to_string().contains("field 7"), "{error}"); - } - - #[test] - fn test_drop_field_removes_it_and_its_data() { - let manifest = backed_manifest(); - let (next, indices) = apply_with_indices( - &manifest, - vec![Action::DropField(DropField { field: 0 })], - vec![sample_index_metadata("idx")], - ) - .unwrap(); - - assert!(next.schema.field_by_id(0).is_none()); - // The file backed only the dropped field, so nothing is left of it. - assert!(next.fragments[0].files.is_empty()); - // An index over a field that no longer exists is discarded outright. - assert!(indices.is_empty()); - } - - #[test] - fn test_drop_field_keeps_a_file_with_a_surviving_field() { - let mut manifest = backed_manifest(); - let mut schema_field = added_field("keep"); - schema_field.id = 1; - manifest.schema.fields.push(schema_field); - let mut fragment = manifest.fragments[0].clone(); - fragment.files[0] = DataFile::new("data/0.lance", vec![0, 1], vec![0, 1], 2, 0, None, None); - manifest.fragments = Arc::new(vec![fragment]); - - let next = apply(&manifest, vec![Action::DropField(DropField { field: 0 })]).unwrap(); - - assert!(next.schema.field_by_id(0).is_none()); - assert!(next.schema.field_by_id(1).is_some()); - // The surviving field stays at the position it was written at. - assert_eq!( - next.fragments[0].files[0].fields.as_ref(), - &[TOMBSTONED_FIELD, 1] - ); - } - - #[test] - fn test_drop_field_takes_the_whole_subtree() { - let mut manifest = backed_manifest(); - let mut parent = Field::try_from(ArrowField::new("parent", DataType::Int32, true)).unwrap(); - parent.id = 1; - let mut child = added_field("child"); - child.id = 2; - child.parent_id = 1; - parent.children.push(child); - manifest.schema.fields.push(parent); - - let next = apply(&manifest, vec![Action::DropField(DropField { field: 1 })]).unwrap(); - - assert!(next.schema.field_by_id(1).is_none()); - assert!( - next.schema.field_by_id(2).is_none(), - "a struct's children cannot outlive it" - ); - } - - #[test] - fn test_drop_field_rejects_a_missing_field() { - let manifest = backed_manifest(); - let error = apply(&manifest, vec![Action::DropField(DropField { field: 7 })]).unwrap_err(); - - assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); - assert!(error.to_string().contains("field 7"), "{error}"); - } - - #[test] - fn test_drop_field_then_add_a_field_reuses_no_id() { - let manifest = backed_manifest(); - let next = apply( - &manifest, - vec![ - Action::DropField(DropField { field: 0 }), - Action::AddField(AddField { - local: 0, - parent: None, - def: added_field("replacement"), - }), - ], - ) - .unwrap(); - - // Field ids come from a monotonic counter, never from the freed id -- - // an old data file naming id 0 must not be read as the new field. - let ids = next - .schema - .fields_pre_order() - .map(|field| field.id) - .collect::>(); - assert_eq!(ids, vec![1]); - } - - #[test] - fn test_add_fragment_and_data_file_mint_ids() { - let manifest = sample_manifest(); - let next = apply( - &manifest, - vec![ - Action::AddFragment(AddFragment { - local: 0, - physical_rows: 10, - row_id_meta: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - data_change: true, - }), - Action::AddDataFile(AddDataFile { - fragment: Ref::Local(0), - file: DataFile::new_unstarted("data/new.lance", 2, 0), - field_ids: vec![Ref::Committed(0)], - data_change: true, - }), - ], - ) - .unwrap(); - - // sample_manifest already holds fragment 0, so the mint lands on 1. - assert_eq!( - next.fragments.iter().map(|f| f.id).collect::>(), - vec![0, 1] - ); - let minted = next.fragments.iter().find(|f| f.id == 1).unwrap(); - assert_eq!(minted.physical_rows, Some(10)); - assert_eq!(minted.files.len(), 1); - // The file's field list is stamped in from the action's refs. - assert_eq!(minted.files[0].fields.as_ref(), &[0]); - assert_eq!(next.max_fragment_id(), Some(1)); - } - - #[test] - fn test_two_add_fields_mint_distinct_ids() { - let manifest = sample_manifest(); - let next = apply( - &manifest, - vec![ - Action::AddField(AddField { - local: 0, - parent: None, - def: added_field("a"), - }), - Action::AddField(AddField { - local: 1, - parent: None, - def: added_field("b"), - }), - ], - ) - .unwrap(); - - let ids = next - .schema - .fields - .iter() - .map(|f| (f.name.as_str(), f.id)) - .collect::>(); - assert_eq!(ids, vec![("id", 0), ("a", 1), ("b", 2)]); - } - - #[test] - fn test_add_field_then_add_its_data_file() { - // The add-column shape: mint the field, then write the file that backs - // it, naming the field by the token the mint has not resolved yet. - let manifest = sample_manifest(); - let next = apply( - &manifest, - vec![ - Action::AddField(AddField { - local: 7, - parent: None, - def: added_field("added"), - }), - Action::AddDataFile(AddDataFile { - fragment: Ref::Committed(0), - file: DataFile::new_unstarted("data/added.lance", 2, 0), - field_ids: vec![Ref::Local(7)], - data_change: true, - }), - ], - ) - .unwrap(); - - let field_id = next.schema.field("added").unwrap().id; - assert_eq!(field_id, 1); - let fragment = next.fragments.iter().find(|f| f.id == 0).unwrap(); - assert_eq!(fragment.files.last().unwrap().fields.as_ref(), &[field_id]); - } - - #[test] - fn test_add_field_under_a_parent() { - let manifest = sample_manifest(); - let next = apply( - &manifest, - vec![ - Action::AddField(AddField { - local: 0, - parent: None, - def: Field::try_from(ArrowField::new( - "nested", - DataType::Struct(Default::default()), - true, - )) - .unwrap(), - }), - Action::AddField(AddField { - local: 1, - parent: Some(Ref::Local(0)), - def: added_field("child"), - }), - ], - ) - .unwrap(); - - let parent = next.schema.field("nested").unwrap(); - assert_eq!(parent.children.len(), 1); - assert_eq!(parent.children[0].name, "child"); - assert_eq!(parent.children[0].parent_id, parent.id); - } - - #[test] - fn test_add_base_mints_an_id_and_rejects_duplicates() { - let manifest = sample_manifest(); - let next = apply( - &manifest, - vec![Action::AddBase(AddBase { - local: 0, - base: BasePath::new(0, "s3://bucket/a".into(), Some("a".into()), false), - })], - ) - .unwrap(); - assert_eq!(next.base_paths.len(), 1); - assert_eq!(next.base_paths[&1].path, "s3://bucket/a"); - - let error = apply( - &next, - vec![Action::AddBase(AddBase { - local: 0, - base: BasePath::new(0, "s3://bucket/a".into(), Some("other".into()), false), - })], - ) - .unwrap_err(); - assert!( - error.to_string().contains("already exists"), - "unexpected error: {error}" - ); - } - - #[test] - fn test_unbound_local_token_is_rejected() { - let manifest = sample_manifest(); - let error = apply( - &manifest, - vec![Action::AddDataFile(AddDataFile { - fragment: Ref::Local(3), - file: DataFile::new_unstarted("data/x.lance", 2, 0), - field_ids: vec![Ref::Committed(0)], - data_change: true, - })], - ) - .unwrap_err(); - assert!( - error.to_string().contains("local fragment token 3"), - "unexpected error: {error}" - ); - } - - #[test] - fn test_duplicate_local_token_is_rejected() { - let manifest = sample_manifest(); - let error = apply( - &manifest, - vec![ - Action::AddField(AddField { - local: 0, - parent: None, - def: added_field("a"), - }), - Action::AddField(AddField { - local: 0, - parent: None, - def: added_field("b"), - }), - ], - ) - .unwrap_err(); - assert!( - error.to_string().contains("minted more than once"), - "unexpected error: {error}" - ); - } + use crate::transaction::action::test_support::{added_field, apply, backed_manifest}; + use crate::transaction::action::{Action, AddDataFile, AddField, AddFragment}; + use crate::transaction::test_support::default_build_config; #[test] fn test_an_action_set_relocates_onto_a_newer_version() { diff --git a/rust/lance-table/src/transaction/action/drop_field.rs b/rust/lance-table/src/transaction/action/drop_field.rs new file mode 100644 index 00000000000..4b03d82479f --- /dev/null +++ b/rust/lance-table/src/transaction/action/drop_field.rs @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Remove a field from the schema. + +use super::Footprint; +use super::apply::{ApplyState, TOMBSTONED_FIELD}; +use super::proto::field_id_from_wire; +use crate::format::pb; +use lance_core::datatypes::Field; +use lance_core::deepsize::DeepSizeOf; +use lance_core::{Error, Result}; +use std::collections::HashSet; + +/// Remove a field from the schema. +/// +/// The field's descendants go with it, since a struct's children cannot outlive +/// it. At apply, any data file left backing no live field is dropped, and any +/// index over a removed field is discarded. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct DropField { + pub field: i32, +} + +impl DropField { + pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { + let field = state.schema().field_by_id(self.field).ok_or_else(|| { + Error::invalid_input(format!( + "DropField names field {}, which does not exist", + self.field + )) + })?; + // A struct's children cannot outlive it, so the whole subtree goes. + let mut dropped = HashSet::new(); + collect_subtree_ids(field, &mut dropped); + remove_field(&mut state.schema_mut().fields, self.field); + + // The fields are gone from the schema, so the slots that backed them in + // each data file are dead. Tombstoning rather than rewriting the field + // list keeps a file's remaining columns at the positions they were + // written at; a file left with nothing live is pruned during + // normalization. + for fragment in state.fragments_mut() { + for file in fragment.files.iter_mut() { + if !file.fields.iter().any(|id| dropped.contains(id)) { + continue; + } + let fields = file + .fields + .iter() + .map(|id| { + if dropped.contains(id) { + TOMBSTONED_FIELD + } else { + *id + } + }) + .collect::>(); + file.fields = fields.into(); + } + + let overlaid: Vec = dropped + .iter() + .filter_map(|id| u32::try_from(*id).ok()) + .collect(); + crate::format::overlay::tombstone_overlay_fields(&mut fragment.overlays, &overlaid); + } + + // Indices over a field that no longer exists are discarded wholesale by + // `retain_relevant_indices`, so there is nothing to record here. + Ok(()) + } + + /// The field's definition and all of its data, which cannot be enumerated, + /// so the removal is recorded as such and matched by field id. + pub(super) fn footprint(&self, footprint: &mut Footprint) { + footprint.remove_field(self.field); + } +} + +fn collect_subtree_ids(field: &Field, out: &mut HashSet) { + out.insert(field.id); + for child in &field.children { + collect_subtree_ids(child, out); + } +} + +/// Remove the field with `field_id` from `fields`, at whatever depth it sits. +fn remove_field(fields: &mut Vec, field_id: i32) { + let before = fields.len(); + fields.retain(|field| field.id != field_id); + if fields.len() != before { + return; + } + for field in fields.iter_mut() { + remove_field(&mut field.children, field_id); + } +} + +impl From<&DropField> for pb::DropField { + fn from(value: &DropField) -> Self { + Self { + field: value.field as u64, + } + } +} + +impl TryFrom for DropField { + type Error = Error; + + fn try_from(message: pb::DropField) -> Result { + Ok(Self { + field: field_id_from_wire(message.field)?, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::DataFile; + use crate::transaction::action::test_support::{ + added_field, apply, apply_with_indices, backed_manifest, + }; + use crate::transaction::action::{Action, AddField}; + use crate::transaction::test_support::sample_index_metadata; + use arrow_schema::{DataType, Field as ArrowField}; + use std::sync::Arc; + + #[test] + fn test_drop_field_removes_it_and_its_data() { + let (next, indices) = apply_with_indices( + &backed_manifest(), + vec![Action::DropField(DropField { field: 0 })], + vec![sample_index_metadata("idx")], + ) + .unwrap(); + + assert!(next.schema.field_by_id(0).is_none()); + // The file backed only the dropped field, so nothing is left of it. + assert!(next.fragments[0].files.is_empty()); + // An index over a field that no longer exists is discarded outright. + assert!(indices.is_empty()); + } + + #[test] + fn test_drop_field_keeps_a_file_with_a_surviving_field() { + let mut manifest = backed_manifest(); + let mut schema_field = added_field("keep"); + schema_field.id = 1; + manifest.schema.fields.push(schema_field); + let mut fragment = manifest.fragments[0].clone(); + fragment.files[0] = DataFile::new("data/0.lance", vec![0, 1], vec![0, 1], 2, 0, None, None); + manifest.fragments = Arc::new(vec![fragment]); + + let next = apply(&manifest, vec![Action::DropField(DropField { field: 0 })]).unwrap(); + + assert!(next.schema.field_by_id(0).is_none()); + assert!(next.schema.field_by_id(1).is_some()); + // The surviving field stays at the position it was written at. + assert_eq!( + next.fragments[0].files[0].fields.as_ref(), + &[TOMBSTONED_FIELD, 1] + ); + } + + #[test] + fn test_drop_field_takes_the_whole_subtree() { + let mut manifest = backed_manifest(); + let mut parent = Field::try_from(ArrowField::new("parent", DataType::Int32, true)).unwrap(); + parent.id = 1; + let mut child = added_field("child"); + child.id = 2; + child.parent_id = 1; + parent.children.push(child); + manifest.schema.fields.push(parent); + + let next = apply(&manifest, vec![Action::DropField(DropField { field: 1 })]).unwrap(); + + assert!(next.schema.field_by_id(1).is_none()); + assert!( + next.schema.field_by_id(2).is_none(), + "a struct's children cannot outlive it" + ); + } + + #[test] + fn test_drop_field_rejects_a_missing_field() { + let error = apply( + &backed_manifest(), + vec![Action::DropField(DropField { field: 7 })], + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + assert!(error.to_string().contains("field 7"), "{error}"); + } + + #[test] + fn test_drop_field_then_add_a_field_reuses_no_id() { + let next = apply( + &backed_manifest(), + vec![ + Action::DropField(DropField { field: 0 }), + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("replacement"), + }), + ], + ) + .unwrap(); + + // Field ids come from a monotonic counter, never from the freed id -- + // an old data file naming id 0 must not be read as the new field. + let ids = next + .schema + .fields_pre_order() + .map(|field| field.id) + .collect::>(); + assert_eq!(ids, vec![1]); + } +} diff --git a/rust/lance-table/src/transaction/action/footprint.rs b/rust/lance-table/src/transaction/action/footprint.rs index 3eb2a3d7b18..822a8ecfd08 100644 --- a/rust/lance-table/src/transaction/action/footprint.rs +++ b/rust/lance-table/src/transaction/action/footprint.rs @@ -12,8 +12,11 @@ //! Footprints are derived from the actions at conflict time and never //! serialized. A writer cannot pin down what a reader considers a conflict, and //! the rule can be tightened in a later release without a format change. +//! +//! Which coordinates an action writes is decided by that action, in its own +//! module. This module holds the coordinate space and the comparison. -use super::{Action, Ref, UserOperation}; +use super::{Ref, UserOperation}; use std::collections::HashSet; /// One thing an action set writes. @@ -99,11 +102,13 @@ impl Footprint { }) } - fn add(&mut self, coordinate: Coordinate) { + pub(super) fn add(&mut self, coordinate: Coordinate) { self.writes.insert(coordinate); } - fn add_field_data(&mut self, fragment: Ref, fields: impl IntoIterator) { + /// The data of each field within `fragment`. A fragment minted in this same + /// operation records nothing: no concurrent writer can be naming it. + pub(super) fn add_field_data(&mut self, fragment: Ref, fields: impl IntoIterator) { let Some(fragment) = fragment.committed() else { return; }; @@ -111,45 +116,23 @@ impl Footprint { self.add(Coordinate::FieldData { fragment, field }); } } + + pub(super) fn remove_fragment(&mut self, fragment: u64) { + self.add(Coordinate::FragmentExistence(fragment)); + self.removed_fragments.insert(fragment); + } + + pub(super) fn remove_field(&mut self, field: i32) { + self.add(Coordinate::FieldDefinition(field)); + self.removed_fields.insert(field); + } } impl From<&UserOperation> for Footprint { fn from(user_operation: &UserOperation) -> Self { let mut footprint = Self::default(); for action in user_operation.iter_actions() { - match action { - // Minting actions name nothing that exists in the read version. - Action::AddFragment(_) | Action::AddField(_) => {} - Action::AddBase(action) => { - footprint.add(Coordinate::BaseName(action.base.name.clone())); - footprint.add(Coordinate::BaseLocation(action.base.path.clone())); - } - Action::AddDataFile(action) => footprint.add_field_data( - action.fragment, - action.field_ids.iter().filter_map(|field| { - field.committed().and_then(|id| i32::try_from(id).ok()) - }), - ), - Action::TombstoneFieldData(action) => { - footprint.add_field_data(action.fragment, action.field_ids.iter().copied()) - } - Action::RemoveFragment(action) => { - if let Some(id) = action.fragment.committed() { - footprint.add(Coordinate::FragmentExistence(id)); - footprint.removed_fragments.insert(id); - } - } - Action::SetDeletionFile(action) => { - footprint.add(Coordinate::FragmentDeletions(action.fragment)) - } - Action::AlterField(action) => { - footprint.add(Coordinate::FieldDefinition(action.field)) - } - Action::DropField(action) => { - footprint.add(Coordinate::FieldDefinition(action.field)); - footprint.removed_fields.insert(action.field); - } - } + action.footprint(&mut footprint); } footprint } @@ -160,7 +143,7 @@ mod tests { use super::*; use crate::format::{BasePath, DataFile}; use crate::transaction::action::{ - AddBase, AddDataFile, AddField, AddFragment, AlterField, DropField, RemoveFragment, + Action, AddBase, AddDataFile, AddField, AddFragment, AlterField, DropField, RemoveFragment, SetDeletionFile, TombstoneFieldData, UserAction, }; use arrow_schema::{DataType, Field as ArrowField}; diff --git a/rust/lance-table/src/transaction/action/proto.rs b/rust/lance-table/src/transaction/action/proto.rs index ef11b5ea8a4..2fd67074bc3 100644 --- a/rust/lance-table/src/transaction/action/proto.rs +++ b/rust/lance-table/src/transaction/action/proto.rs @@ -1,25 +1,23 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -//! Conversions between the action vocabulary and its protobuf encoding. +//! The envelope around the per-action protobuf encodings. +//! +//! Each action encodes itself, in its own module; this module carries the +//! [`Ref`], [`UserOperation`], and [`UserAction`] wrappers, the dispatch over +//! the `oneof`, and the helpers the per-action conversions share. //! //! Reading is fail-closed: an action this build does not implement is an error, //! never a silently skipped element. The commit path collects concurrent //! transactions with `try_collect`, so a transaction carrying an unknown action //! must abort the commit rather than be treated as a no-op. -use super::{ - Action, AddBase, AddDataFile, AddField, AddFragment, AlterField, DropField, Ref, - RemoveFragment, SetDeletionFile, TombstoneFieldData, UserAction, UserOperation, -}; +use super::{Action, Ref, UserAction, UserOperation}; use crate::format::pb; -use crate::format::{BasePath, DataFile, DeletionFile, ExternalFile, RowIdMeta}; -use crate::rowids::version::RowDatasetVersionMeta; -use lance_core::datatypes::Field; use lance_core::{Error, Result}; /// A field id on the wire is a `uint64`; in the manifest it is an `i32`. -fn field_id_from_wire(id: u64) -> Result { +pub(super) fn field_id_from_wire(id: u64) -> Result { i32::try_from(id).map_err(|_| { Error::invalid_input(format!( "field id {id} in an action exceeds the maximum field id ({})", @@ -28,6 +26,20 @@ fn field_id_from_wire(id: u64) -> Result { }) } +/// `data_change` is absent-means-true on the wire, so only the `false` case is +/// written out. +pub(super) fn data_change_to_wire(data_change: bool) -> Option { + (!data_change).then_some(false) +} + +pub(super) fn data_change_from_wire(data_change: Option) -> bool { + data_change.unwrap_or(true) +} + +pub(super) fn required(value: Option, what: &str) -> Result { + value.ok_or_else(|| Error::invalid_input(format!("{what} is required but was not set"))) +} + impl From for pb::Ref { fn from(value: Ref) -> Self { let kind = match value { @@ -52,16 +64,6 @@ impl TryFrom for Ref { } } -/// `data_change` is absent-means-true on the wire, so only the `false` case is -/// written out. -fn data_change_to_wire(data_change: bool) -> Option { - (!data_change).then_some(false) -} - -fn data_change_from_wire(data_change: Option) -> bool { - data_change.unwrap_or(true) -} - impl From<&UserOperation> for pb::UserOperation { fn from(value: &UserOperation) -> Self { Self { @@ -175,302 +177,17 @@ impl TryFrom for Action { } } -impl From<&AddFragment> for pb::AddFragment { - fn from(value: &AddFragment) -> Self { - Self { - local: value.local, - physical_rows: value.physical_rows, - row_id_sequence: value.row_id_meta.as_ref().map(|meta| match meta { - RowIdMeta::Inline(data) => { - pb::add_fragment::RowIdSequence::InlineRowIds(data.clone()) - } - RowIdMeta::External(file) => { - pb::add_fragment::RowIdSequence::ExternalRowIds(external_file_to_wire(file)) - } - }), - last_updated_at_version_sequence: value - .last_updated_at_version_meta - .as_ref() - .map(|meta| match meta { - RowDatasetVersionMeta::Inline(data) => { - pb::add_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions( - data.to_vec(), - ) - } - RowDatasetVersionMeta::External(file) => { - pb::add_fragment::LastUpdatedAtVersionSequence::ExternalLastUpdatedAtVersions( - external_file_to_wire(file), - ) - } - }), - created_at_version_sequence: value - .created_at_version_meta - .as_ref() - .map(|meta| match meta { - RowDatasetVersionMeta::Inline(data) => { - pb::add_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions( - data.to_vec(), - ) - } - RowDatasetVersionMeta::External(file) => { - pb::add_fragment::CreatedAtVersionSequence::ExternalCreatedAtVersions( - external_file_to_wire(file), - ) - } - }), - data_change: data_change_to_wire(value.data_change), - } - } -} - -impl TryFrom for AddFragment { - type Error = Error; - - fn try_from(message: pb::AddFragment) -> Result { - Ok(Self { - local: message.local, - physical_rows: message.physical_rows, - row_id_meta: message.row_id_sequence.map(|sequence| match sequence { - pb::add_fragment::RowIdSequence::InlineRowIds(data) => RowIdMeta::Inline(data), - pb::add_fragment::RowIdSequence::ExternalRowIds(file) => { - RowIdMeta::External(external_file_from_wire(file)) - } - }), - last_updated_at_version_meta: message.last_updated_at_version_sequence.map( - |sequence| { - match sequence { - pb::add_fragment::LastUpdatedAtVersionSequence::InlineLastUpdatedAtVersions( - data, - ) => RowDatasetVersionMeta::Inline(data.into()), - pb::add_fragment::LastUpdatedAtVersionSequence::ExternalLastUpdatedAtVersions( - file, - ) => RowDatasetVersionMeta::External(external_file_from_wire(file)), - } - }, - ), - created_at_version_meta: message.created_at_version_sequence.map(|sequence| { - match sequence { - pb::add_fragment::CreatedAtVersionSequence::InlineCreatedAtVersions(data) => { - RowDatasetVersionMeta::Inline(data.into()) - } - pb::add_fragment::CreatedAtVersionSequence::ExternalCreatedAtVersions(file) => { - RowDatasetVersionMeta::External(external_file_from_wire(file)) - } - } - }), - data_change: data_change_from_wire(message.data_change), - }) - } -} - -fn external_file_to_wire(file: &ExternalFile) -> pb::ExternalFile { - pb::ExternalFile { - path: file.path.clone(), - offset: file.offset, - size: file.size, - } -} - -fn external_file_from_wire(file: pb::ExternalFile) -> ExternalFile { - ExternalFile { - path: file.path, - offset: file.offset, - size: file.size, - } -} - -impl From<&AddDataFile> for pb::AddDataFile { - fn from(value: &AddDataFile) -> Self { - Self { - fragment: Some(value.fragment.into()), - file: Some(pb::DataFile::from(&value.file)), - field_ids: value.field_ids.iter().map(|id| (*id).into()).collect(), - data_change: data_change_to_wire(value.data_change), - } - } -} - -impl TryFrom for AddDataFile { - type Error = Error; - - fn try_from(message: pb::AddDataFile) -> Result { - Ok(Self { - fragment: required(message.fragment, "AddDataFile.fragment")?.try_into()?, - file: DataFile::try_from(required(message.file, "AddDataFile.file")?)?, - field_ids: message - .field_ids - .into_iter() - .map(Ref::try_from) - .collect::>>()?, - data_change: data_change_from_wire(message.data_change), - }) - } -} - -impl From<&AddField> for pb::AddField { - fn from(value: &AddField) -> Self { - Self { - local: value.local, - parent: value.parent.map(Into::into), - def: Some(lance_file::format::pb::Field::from(&value.def)), - } - } -} - -impl TryFrom for AddField { - type Error = Error; - - fn try_from(message: pb::AddField) -> Result { - Ok(Self { - local: message.local, - parent: message.parent.map(Ref::try_from).transpose()?, - def: Field::from(&required(message.def, "AddField.def")?), - }) - } -} - -impl From<&AddBase> for pb::AddBase { - fn from(value: &AddBase) -> Self { - Self { - local: value.local, - base: Some(pb::BasePath::from(value.base.clone())), - } - } -} - -impl TryFrom for AddBase { - type Error = Error; - - fn try_from(message: pb::AddBase) -> Result { - Ok(Self { - local: message.local, - base: BasePath::from(required(message.base, "AddBase.base")?), - }) - } -} - -impl From<&TombstoneFieldData> for pb::TombstoneFieldData { - fn from(value: &TombstoneFieldData) -> Self { - Self { - fragment: Some(value.fragment.into()), - field_ids: value.field_ids.iter().map(|id| *id as u64).collect(), - data_change: data_change_to_wire(value.data_change), - } - } -} - -impl TryFrom for TombstoneFieldData { - type Error = Error; - - fn try_from(message: pb::TombstoneFieldData) -> Result { - Ok(Self { - fragment: required(message.fragment, "TombstoneFieldData.fragment")?.try_into()?, - field_ids: message - .field_ids - .into_iter() - .map(field_id_from_wire) - .collect::>>()?, - data_change: data_change_from_wire(message.data_change), - }) - } -} - -impl From<&RemoveFragment> for pb::RemoveFragment { - fn from(value: &RemoveFragment) -> Self { - Self { - fragment: Some(value.fragment.into()), - data_change: data_change_to_wire(value.data_change), - } - } -} - -impl TryFrom for RemoveFragment { - type Error = Error; - - fn try_from(message: pb::RemoveFragment) -> Result { - Ok(Self { - fragment: required(message.fragment, "RemoveFragment.fragment")?.try_into()?, - data_change: data_change_from_wire(message.data_change), - }) - } -} - -impl From<&SetDeletionFile> for pb::SetDeletionFile { - fn from(value: &SetDeletionFile) -> Self { - Self { - fragment: value.fragment, - deletion_file: value.deletion_file.as_ref().map(pb::DeletionFile::from), - data_change: data_change_to_wire(value.data_change), - } - } -} - -impl TryFrom for SetDeletionFile { - type Error = Error; - - fn try_from(message: pb::SetDeletionFile) -> Result { - Ok(Self { - fragment: message.fragment, - deletion_file: message - .deletion_file - .map(DeletionFile::try_from) - .transpose()?, - data_change: data_change_from_wire(message.data_change), - }) - } -} - -impl From<&AlterField> for pb::AlterField { - fn from(value: &AlterField) -> Self { - Self { - field: value.field as u64, - name: value.name.clone(), - logical_type: value.logical_type.clone(), - nullable: value.nullable, - } - } -} - -impl TryFrom for AlterField { - type Error = Error; - - fn try_from(message: pb::AlterField) -> Result { - Ok(Self { - field: field_id_from_wire(message.field)?, - name: message.name, - logical_type: message.logical_type, - nullable: message.nullable, - }) - } -} - -impl From<&DropField> for pb::DropField { - fn from(value: &DropField) -> Self { - Self { - field: value.field as u64, - } - } -} - -impl TryFrom for DropField { - type Error = Error; - - fn try_from(message: pb::DropField) -> Result { - Ok(Self { - field: field_id_from_wire(message.field)?, - }) - } -} - -fn required(value: Option, what: &str) -> Result { - value.ok_or_else(|| Error::invalid_input(format!("{what} is required but was not set"))) -} - #[cfg(test)] mod tests { use super::*; - use crate::format::{DeletionFileType, pb}; + use crate::format::{BasePath, DataFile, DeletionFile, DeletionFileType, RowIdMeta, pb}; + use crate::rowids::version::RowDatasetVersionMeta; + use crate::transaction::action::{ + AddBase, AddDataFile, AddField, AddFragment, AlterField, DropField, RemoveFragment, + SetDeletionFile, TombstoneFieldData, + }; use arrow_schema::{DataType, Field as ArrowField}; + use lance_core::datatypes::Field; use std::sync::Arc; fn sample_data_file() -> DataFile { @@ -595,4 +312,13 @@ mod tests { "unexpected message: {error}" ); } + + #[test] + fn test_field_id_out_of_range_is_rejected() { + let error = field_id_from_wire(u64::from(u32::MAX) + 1).unwrap_err(); + assert!( + error.to_string().contains("exceeds the maximum field id"), + "unexpected message: {error}" + ); + } } diff --git a/rust/lance-table/src/transaction/action/remove_fragment.rs b/rust/lance-table/src/transaction/action/remove_fragment.rs new file mode 100644 index 00000000000..65955398bed --- /dev/null +++ b/rust/lance-table/src/transaction/action/remove_fragment.rs @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Remove a fragment entirely. + +use super::apply::ApplyState; +use super::proto::{data_change_from_wire, data_change_to_wire, required}; +use super::{Footprint, Ref}; +use crate::format::pb; +use lance_core::deepsize::DeepSizeOf; +use lance_core::{Error, Result}; + +/// Remove a fragment entirely -- every row deleted, or the fragment replaced by +/// compaction. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct RemoveFragment { + pub fragment: Ref, + /// See [`AddFragment::data_change`](super::AddFragment::data_change). + pub data_change: bool, +} + +impl RemoveFragment { + pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { + let fragment_id = state.resolve_fragment(self.fragment)?; + if !state.remove_fragment(fragment_id) { + return Err(Error::invalid_input(format!( + "RemoveFragment targets fragment {fragment_id}, which does not exist" + ))); + } + Ok(()) + } + + /// Every coordinate inside the fragment, which cannot be enumerated, so the + /// removal is recorded as such and matched by fragment id. + pub(super) fn footprint(&self, footprint: &mut Footprint) { + if let Some(id) = self.fragment.committed() { + footprint.remove_fragment(id); + } + } +} + +impl From<&RemoveFragment> for pb::RemoveFragment { + fn from(value: &RemoveFragment) -> Self { + Self { + fragment: Some(value.fragment.into()), + data_change: data_change_to_wire(value.data_change), + } + } +} + +impl TryFrom for RemoveFragment { + type Error = Error; + + fn try_from(message: pb::RemoveFragment) -> Result { + Ok(Self { + fragment: required(message.fragment, "RemoveFragment.fragment")?.try_into()?, + data_change: data_change_from_wire(message.data_change), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::transaction::action::test_support::{apply, backed_manifest}; + use crate::transaction::action::{Action, AddFragment}; + + #[test] + fn test_remove_fragment() { + let next = apply( + &backed_manifest(), + vec![Action::RemoveFragment(RemoveFragment { + fragment: Ref::Committed(0), + data_change: true, + })], + ) + .unwrap(); + + assert!(next.fragments.is_empty()); + } + + #[test] + fn test_remove_fragment_can_drop_one_minted_in_the_same_operation() { + let next = apply( + &backed_manifest(), + vec![ + Action::AddFragment(AddFragment { + local: 0, + physical_rows: 10, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }), + Action::RemoveFragment(RemoveFragment { + fragment: Ref::Local(0), + data_change: true, + }), + ], + ) + .unwrap(); + + assert_eq!( + next.fragments.iter().map(|f| f.id).collect::>(), + vec![0] + ); + } + + #[test] + fn test_remove_fragment_rejects_a_missing_fragment() { + let error = apply( + &backed_manifest(), + vec![Action::RemoveFragment(RemoveFragment { + fragment: Ref::Committed(7), + data_change: true, + })], + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + assert!(error.to_string().contains("fragment 7"), "{error}"); + } +} diff --git a/rust/lance-table/src/transaction/action/set_deletion_file.rs b/rust/lance-table/src/transaction/action/set_deletion_file.rs new file mode 100644 index 00000000000..4406681c83e --- /dev/null +++ b/rust/lance-table/src/transaction/action/set_deletion_file.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Set (replace) a fragment's deletion file. + +use super::apply::ApplyState; +use super::proto::{data_change_from_wire, data_change_to_wire}; +use super::{Coordinate, Footprint}; +use crate::format::{DeletionFile, pb}; +use lance_core::deepsize::DeepSizeOf; +use lance_core::{Error, Result}; + +/// Set (replace) a fragment's deletion file. +/// +/// This is reference-stable rather than a delta: the fragment id is committed +/// and physical row offsets never move, so the post-image is unambiguous. The +/// newly-deleted rows -- the delta rebase and conflict detection need -- are +/// derived by diffing against the read version rather than serialized. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct SetDeletionFile { + /// The fragment, by committed id. Unlike its sibling fragment actions this + /// takes no [`Ref`](super::Ref): a fragment minted in the same operation has + /// no committed rows to delete. + pub fragment: u64, + /// The new deletion file, or `None` to clear the fragment's deletions. + pub deletion_file: Option, + /// See [`AddFragment::data_change`](super::AddFragment::data_change). + pub data_change: bool, +} + +impl SetDeletionFile { + pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { + state + .fragment_mut(self.fragment, "SetDeletionFile")? + .deletion_file = self.deletion_file.clone(); + Ok(()) + } + + /// The fragment's deletions, which is a distinct coordinate from the data of + /// any field in it: deleting rows and re-encoding a column commute. + pub(super) fn footprint(&self, footprint: &mut Footprint) { + footprint.add(Coordinate::FragmentDeletions(self.fragment)); + } +} + +impl From<&SetDeletionFile> for pb::SetDeletionFile { + fn from(value: &SetDeletionFile) -> Self { + Self { + fragment: value.fragment, + deletion_file: value.deletion_file.as_ref().map(pb::DeletionFile::from), + data_change: data_change_to_wire(value.data_change), + } + } +} + +impl TryFrom for SetDeletionFile { + type Error = Error; + + fn try_from(message: pb::SetDeletionFile) -> Result { + Ok(Self { + fragment: message.fragment, + deletion_file: message + .deletion_file + .map(DeletionFile::try_from) + .transpose()?, + data_change: data_change_from_wire(message.data_change), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::DeletionFileType; + use crate::transaction::action::Action; + use crate::transaction::action::test_support::{apply, backed_manifest}; + + #[test] + fn test_set_deletion_file_sets_and_clears() { + let manifest = backed_manifest(); + let deletion_file = DeletionFile { + read_version: manifest.version, + id: 3, + file_type: DeletionFileType::Array, + num_deleted_rows: Some(2), + base_id: None, + }; + let next = apply( + &manifest, + vec![Action::SetDeletionFile(SetDeletionFile { + fragment: 0, + deletion_file: Some(deletion_file.clone()), + data_change: true, + })], + ) + .unwrap(); + assert_eq!(next.fragments[0].deletion_file, Some(deletion_file)); + + // An absent deletion file is a request to clear it, not a no-op. + let cleared = apply( + &next, + vec![Action::SetDeletionFile(SetDeletionFile { + fragment: 0, + deletion_file: None, + data_change: true, + })], + ) + .unwrap(); + assert_eq!(cleared.fragments[0].deletion_file, None); + } + + #[test] + fn test_set_deletion_file_rejects_a_missing_fragment() { + let error = apply( + &backed_manifest(), + vec![Action::SetDeletionFile(SetDeletionFile { + fragment: 7, + deletion_file: None, + data_change: true, + })], + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + } +} diff --git a/rust/lance-table/src/transaction/action/test_support.rs b/rust/lance-table/src/transaction/action/test_support.rs new file mode 100644 index 00000000000..080b13c609c --- /dev/null +++ b/rust/lance-table/src/transaction/action/test_support.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Fixtures shared by the per-action test modules. + +use super::{Action, UserAction, UserOperation}; +use crate::format::{DataFile, Fragment, IndexMetadata, Manifest}; +use crate::transaction::test_support::{default_build_config, sample_manifest}; +use crate::transaction::{Operation, Transaction}; +use arrow_schema::{DataType, Field as ArrowField}; +use lance_core::Result; +use lance_core::datatypes::Field; +use std::sync::Arc; + +pub(super) fn apply(manifest: &Manifest, actions: Vec) -> Result { + apply_with_indices(manifest, actions, Vec::new()).map(|(manifest, _)| manifest) +} + +pub(super) fn apply_with_indices( + manifest: &Manifest, + actions: Vec, + indices: Vec, +) -> Result<(Manifest, Vec)> { + let transaction = Transaction::new( + manifest.version, + Operation::UserOperation(UserOperation::new( + "test", + vec![UserAction::new("step", actions)], + )), + None, + ); + transaction.build_manifest(Some(manifest), indices, "tx.txn", &default_build_config()) +} + +pub(super) fn added_field(name: &str) -> Field { + Field::try_from(ArrowField::new(name, DataType::Int32, true)).unwrap() +} + +/// `sample_manifest` with fragment 0 actually backed by a data file, so the +/// reference-stable actions have something committed to point at. +pub(super) fn backed_manifest() -> Manifest { + let mut manifest = sample_manifest(); + let mut fragment = Fragment::new(0); + fragment.physical_rows = Some(10); + fragment.files.push(DataFile::new( + "data/0.lance", + vec![0], + vec![0], + 2, + 0, + None, + None, + )); + manifest.fragments = Arc::new(vec![fragment]); + manifest +} diff --git a/rust/lance-table/src/transaction/action/tombstone_field_data.rs b/rust/lance-table/src/transaction/action/tombstone_field_data.rs new file mode 100644 index 00000000000..59b0e2f8ad1 --- /dev/null +++ b/rust/lance-table/src/transaction/action/tombstone_field_data.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Tombstone the data-file binding of committed fields within one fragment. + +use super::apply::{ApplyState, TOMBSTONED_FIELD}; +use super::proto::{data_change_from_wire, data_change_to_wire, field_id_from_wire, required}; +use super::{Footprint, Ref}; +use crate::format::pb; +use lance_core::deepsize::DeepSizeOf; +use lance_core::{Error, Result}; + +/// Tombstone the data-file binding of committed fields within one fragment. +/// +/// Each field's slot in whatever file currently backs it is marked tombstoned, +/// and a file left with no live field is pruned at apply. Data files have no id +/// of their own and a live field is backed by exactly one file, so this is how a +/// column's data is dropped or superseded: re-encoding a column is a tombstone +/// followed by an [`AddDataFile`](super::AddDataFile) for the same field. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct TombstoneFieldData { + pub fragment: Ref, + /// Committed field ids whose current backing is tombstoned. + pub field_ids: Vec, + /// See [`AddFragment::data_change`](super::AddFragment::data_change). + pub data_change: bool, +} + +impl TombstoneFieldData { + pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { + let fragment_id = state.resolve_fragment(self.fragment)?; + let fragment = state.fragment_mut(fragment_id, "TombstoneFieldData")?; + + for &field_id in &self.field_ids { + let mut found = false; + for file in fragment.files.iter_mut() { + let Some(position) = file.fields.iter().position(|id| *id == field_id) else { + continue; + }; + let mut fields = file.fields.to_vec(); + fields[position] = TOMBSTONED_FIELD; + file.fields = fields.into(); + found = true; + } + if !found { + return Err(Error::invalid_input(format!( + "TombstoneFieldData names field {field_id}, which no data file in fragment \ + {fragment_id} backs" + ))); + } + } + + // New values for these fields supersede any overlay still shadowing + // them, so the drop is not silently masked by stale overlay cells. + let overlaid: Vec = self + .field_ids + .iter() + .filter_map(|id| u32::try_from(*id).ok()) + .collect(); + crate::format::overlay::tombstone_overlay_fields(&mut fragment.overlays, &overlaid); + + state.rebind_fields(fragment_id, self.field_ids.iter().copied()); + Ok(()) + } + + /// The data of each named field in this fragment, and nothing else: another + /// field's data in the same fragment is untouched. + pub(super) fn footprint(&self, footprint: &mut Footprint) { + footprint.add_field_data(self.fragment, self.field_ids.iter().copied()); + } +} + +impl From<&TombstoneFieldData> for pb::TombstoneFieldData { + fn from(value: &TombstoneFieldData) -> Self { + Self { + fragment: Some(value.fragment.into()), + field_ids: value.field_ids.iter().map(|id| *id as u64).collect(), + data_change: data_change_to_wire(value.data_change), + } + } +} + +impl TryFrom for TombstoneFieldData { + type Error = Error; + + fn try_from(message: pb::TombstoneFieldData) -> Result { + Ok(Self { + fragment: required(message.fragment, "TombstoneFieldData.fragment")?.try_into()?, + field_ids: message + .field_ids + .into_iter() + .map(field_id_from_wire) + .collect::>>()?, + data_change: data_change_from_wire(message.data_change), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::DataFile; + use crate::transaction::action::Action; + use crate::transaction::action::test_support::{apply, apply_with_indices, backed_manifest}; + use crate::transaction::test_support::sample_index_metadata; + use std::sync::Arc; + + fn tombstone_field_zero() -> Action { + Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Committed(0), + field_ids: vec![0], + data_change: true, + }) + } + + #[test] + fn test_tombstone_field_data_drops_the_backing_file() { + let next = apply(&backed_manifest(), vec![tombstone_field_zero()]).unwrap(); + + // The file backed only field 0, so tombstoning it leaves nothing behind. + assert!(next.fragments[0].files.is_empty()); + } + + #[test] + fn test_tombstone_field_data_keeps_a_file_with_a_live_field() { + let mut manifest = backed_manifest(); + let mut fragment = manifest.fragments[0].clone(); + fragment.files[0] = DataFile::new("data/0.lance", vec![0, 1], vec![0, 1], 2, 0, None, None); + manifest.fragments = Arc::new(vec![fragment]); + + let next = apply(&manifest, vec![tombstone_field_zero()]).unwrap(); + + assert_eq!( + next.fragments[0].files[0].fields.as_ref(), + &[TOMBSTONED_FIELD, 1] + ); + } + + #[test] + fn test_tombstone_field_data_prunes_the_fragment_from_covering_indices() { + let (_, indices) = apply_with_indices( + &backed_manifest(), + vec![tombstone_field_zero()], + vec![sample_index_metadata("idx")], + ) + .unwrap(); + + // The index covers field 0, whose data in fragment 0 is now gone. + assert!(indices[0].fragment_bitmap.as_ref().unwrap().is_empty()); + } + + #[test] + fn test_tombstone_field_data_rejects_a_field_no_file_backs() { + let error = apply( + &backed_manifest(), + vec![Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Committed(0), + field_ids: vec![7], + data_change: true, + })], + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + assert!(error.to_string().contains("field 7"), "{error}"); + } +} From cbefa2c2de4ded9b07593d80433e3e767e8205f4 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 17 Aug 2026 14:31:10 -0700 Subject: [PATCH 14/24] feat(transaction): add the ReserveFragmentIds action Takes a contiguous range of fragment ids off the counter without minting fragments for them, so a later (possibly distributed) writer can populate the range and name the ids as committed. The counterpart of the legacy ReserveFragments operation. Nothing backs a reserved id, so the manifest assembly cannot infer it from the fragment list; apply raises the manifest's high-water mark to cover the range instead. The range starts wherever the counter stands, which unlike the legacy operation does not waste an id on an empty table. The action writes no coordinates: ids come off a monotonic counter, so two operations reserving at once get disjoint ranges. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance-table/src/transaction/action.rs | 12 +- .../src/transaction/action/apply.rs | 36 ++++++ .../src/transaction/action/proto.rs | 9 +- .../action/reserve_fragment_ids.rs | 117 ++++++++++++++++++ 4 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 rust/lance-table/src/transaction/action/reserve_fragment_ids.rs diff --git a/rust/lance-table/src/transaction/action.rs b/rust/lance-table/src/transaction/action.rs index b4835ce9796..338ea5bdd37 100644 --- a/rust/lance-table/src/transaction/action.rs +++ b/rust/lance-table/src/transaction/action.rs @@ -43,6 +43,7 @@ mod drop_field; mod footprint; mod proto; mod remove_fragment; +mod reserve_fragment_ids; mod set_deletion_file; mod tombstone_field_data; mod translate; @@ -58,6 +59,7 @@ pub use alter_field::AlterField; pub use drop_field::DropField; pub use footprint::{Coordinate, Footprint}; pub use remove_fragment::RemoveFragment; +pub use reserve_fragment_ids::ReserveFragmentIds; pub use set_deletion_file::SetDeletionFile; pub use tombstone_field_data::TombstoneFieldData; @@ -168,6 +170,7 @@ pub enum Action { SetDeletionFile(SetDeletionFile), AlterField(AlterField), DropField(DropField), + ReserveFragmentIds(ReserveFragmentIds), } impl Action { @@ -182,6 +185,7 @@ impl Action { Self::SetDeletionFile(_) => "SetDeletionFile", Self::AlterField(_) => "AlterField", Self::DropField(_) => "DropField", + Self::ReserveFragmentIds(_) => "ReserveFragmentIds", } } @@ -200,7 +204,11 @@ impl Action { // Dropping a field discards the values it held. Self::DropField(_) => true, // Other schema and base-path changes touch no row values. - Self::AddField(_) | Self::AddBase(_) | Self::AlterField(_) => false, + // Reserving ids writes no rows either. + Self::AddField(_) + | Self::AddBase(_) + | Self::AlterField(_) + | Self::ReserveFragmentIds(_) => false, } } @@ -216,6 +224,7 @@ impl Action { Self::SetDeletionFile(action) => action.apply(state), Self::AlterField(action) => action.apply(state), Self::DropField(action) => action.apply(state), + Self::ReserveFragmentIds(action) => action.apply(state), } } @@ -231,6 +240,7 @@ impl Action { Self::SetDeletionFile(action) => action.footprint(footprint), Self::AlterField(action) => action.footprint(footprint), Self::DropField(action) => action.footprint(footprint), + Self::ReserveFragmentIds(action) => action.footprint(footprint), } } } diff --git a/rust/lance-table/src/transaction/action/apply.rs b/rust/lance-table/src/transaction/action/apply.rs index 719b47be2b4..5eb79d4e954 100644 --- a/rust/lance-table/src/transaction/action/apply.rs +++ b/rust/lance-table/src/transaction/action/apply.rs @@ -68,6 +68,7 @@ impl Transaction { mut fragments, new_bases, rebound_fields, + reserved_fragment_ids, .. } = state; @@ -89,6 +90,24 @@ impl Transaction { manifest.base_paths.insert(base.id, base); } + // A reserved id backs no fragment, so the manifest assembly cannot + // derive it from the fragment list; raise the high-water mark to cover + // the range so a later writer's ids are not handed out twice. + if let Some(high_water) = reserved_fragment_ids { + let high_water = u32::try_from(high_water).map_err(|_| { + Error::invalid_input(format!( + "reserving fragment ids up to {high_water} exceeds the maximum fragment id \ + ({})", + u32::MAX + )) + })?; + manifest.max_fragment_id = Some( + manifest + .max_fragment_id + .map_or(high_water, |current| current.max(high_water)), + ); + } + manifest.transaction_file = Some(transaction_file_path.to_string()); if let Some(next_row_id) = next_row_id { manifest.next_row_id = next_row_id; @@ -120,6 +139,11 @@ pub(super) struct ApplyState { /// Ids of the fragments this operation minted. minted_fragments: HashSet, + /// The highest fragment id this operation reserved for a later writer, if + /// it reserved any. No fragment backs it, so the manifest assembly cannot + /// infer it from the fragment list. + reserved_fragment_ids: Option, + /// Fields whose backing data changed, per fragment. An index covering such /// a field no longer describes that fragment's contents. rebound_fields: HashMap>, @@ -144,6 +168,7 @@ impl ApplyState { field_tokens: HashMap::new(), base_tokens: HashMap::new(), minted_fragments: HashSet::new(), + reserved_fragment_ids: None, rebound_fields: HashMap::new(), } } @@ -211,6 +236,17 @@ impl ApplyState { Ok(id) } + /// Take `count` ids off the fragment counter without minting fragments for + /// them. The reserved ids are `[next, next + count)`; a later writer names + /// them as [`Ref::Committed`]. + pub(super) fn reserve_fragment_ids(&mut self, count: u32) { + if count == 0 { + return; + } + self.next_fragment_id += u64::from(count); + self.reserved_fragment_ids = Some(self.next_fragment_id - 1); + } + pub(super) fn mint_field(&mut self, token: u32) -> Result { if self.field_tokens.contains_key(&token) { return Err(duplicate_token_err("field", token)); diff --git a/rust/lance-table/src/transaction/action/proto.rs b/rust/lance-table/src/transaction/action/proto.rs index 2fd67074bc3..9868d0f532c 100644 --- a/rust/lance-table/src/transaction/action/proto.rs +++ b/rust/lance-table/src/transaction/action/proto.rs @@ -130,6 +130,9 @@ impl From<&Action> for pb::Action { Action::SetDeletionFile(action) => pb::action::Action::SetDeletionFile(action.into()), Action::AlterField(action) => pb::action::Action::AlterField(action.into()), Action::DropField(action) => pb::action::Action::DropField(action.into()), + Action::ReserveFragmentIds(action) => { + pb::action::Action::ReserveFragmentIds(action.into()) + } }; Self { action: Some(action), @@ -163,6 +166,9 @@ impl TryFrom for Action { Ok(Self::AlterField(action.try_into()?)) } Some(pb::action::Action::DropField(action)) => Ok(Self::DropField(action.try_into()?)), + Some(pb::action::Action::ReserveFragmentIds(action)) => { + Ok(Self::ReserveFragmentIds(action.try_into()?)) + } // The drafted vocabulary is larger than what is implemented. Reject // rather than skip: silently dropping an action would apply a // partial transaction. @@ -184,7 +190,7 @@ mod tests { use crate::rowids::version::RowDatasetVersionMeta; use crate::transaction::action::{ AddBase, AddDataFile, AddField, AddFragment, AlterField, DropField, RemoveFragment, - SetDeletionFile, TombstoneFieldData, + ReserveFragmentIds, SetDeletionFile, TombstoneFieldData, }; use arrow_schema::{DataType, Field as ArrowField}; use lance_core::datatypes::Field; @@ -248,6 +254,7 @@ mod tests { nullable: Some(false), }), Action::DropField(DropField { field: 3 }), + Action::ReserveFragmentIds(ReserveFragmentIds { count: 4 }), ] } diff --git a/rust/lance-table/src/transaction/action/reserve_fragment_ids.rs b/rust/lance-table/src/transaction/action/reserve_fragment_ids.rs new file mode 100644 index 00000000000..d5c71e4704c --- /dev/null +++ b/rust/lance-table/src/transaction/action/reserve_fragment_ids.rs @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Reserve a range of fragment ids for a later writer. + +use super::Footprint; +use super::apply::ApplyState; +use crate::format::pb; +use lance_core::deepsize::DeepSizeOf; +use lance_core::{Error, Result}; + +/// Reserve a contiguous range of fragment ids from the counter, for a later +/// (possibly distributed) writer to populate. +/// +/// The range starts wherever the counter stands when this action is applied, so +/// the reserving writer learns which ids it got by reading the committed +/// manifest's high-water mark: the range is the `count` ids ending there. +/// Fragments written against the range name those ids as +/// [`Ref::Committed`](super::Ref::Committed). +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct ReserveFragmentIds { + pub count: u32, +} + +impl ReserveFragmentIds { + pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { + state.reserve_fragment_ids(self.count); + Ok(()) + } + + /// Nothing. Ids come off a monotonic counter, so two operations reserving + /// at once get disjoint ranges rather than colliding. + pub(super) fn footprint(&self, _footprint: &mut Footprint) {} +} + +impl From<&ReserveFragmentIds> for pb::ReserveFragmentIds { + fn from(value: &ReserveFragmentIds) -> Self { + Self { count: value.count } + } +} + +impl TryFrom for ReserveFragmentIds { + type Error = Error; + + fn try_from(message: pb::ReserveFragmentIds) -> Result { + Ok(Self { + count: message.count, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::transaction::action::test_support::{apply, backed_manifest}; + use crate::transaction::action::{Action, AddFragment}; + + fn reserve(count: u32) -> Action { + Action::ReserveFragmentIds(ReserveFragmentIds { count }) + } + + fn add_fragment(local: u32) -> Action { + Action::AddFragment(AddFragment { + local, + physical_rows: 10, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }) + } + + #[test] + fn test_reserving_raises_the_high_water_mark_without_adding_fragments() { + let manifest = backed_manifest(); + assert_eq!(manifest.max_fragment_id(), Some(0)); + + let next = apply(&manifest, vec![reserve(3)]).unwrap(); + + // Ids 1, 2 and 3 are now spoken for, but no fragment exists for them. + assert_eq!(next.max_fragment_id(), Some(3)); + assert_eq!(next.fragments.len(), 1); + } + + #[test] + fn test_a_later_mint_skips_the_reserved_range() { + let next = apply(&backed_manifest(), vec![reserve(3), add_fragment(0)]).unwrap(); + + assert_eq!( + next.fragments.iter().map(|f| f.id).collect::>(), + vec![0, 4], + "the minted fragment must not land inside the reserved range" + ); + } + + #[test] + fn test_a_reserved_id_is_usable_by_the_next_operation() { + let reserved = apply(&backed_manifest(), vec![reserve(2)]).unwrap(); + assert_eq!(reserved.max_fragment_id(), Some(2)); + + // The next operation mints past the range rather than into it: the + // reservation holds even though nothing was written against it. + let next = apply(&reserved, vec![add_fragment(0)]).unwrap(); + assert_eq!( + next.fragments.iter().map(|f| f.id).collect::>(), + vec![0, 3] + ); + } + + #[test] + fn test_reserving_nothing_is_a_no_op() { + let manifest = backed_manifest(); + let next = apply(&manifest, vec![reserve(0)]).unwrap(); + + assert_eq!(next.max_fragment_id(), manifest.max_fragment_id()); + } +} From 5b66944ada4c486c0a011178b620355eeb28a19c Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 17 Aug 2026 14:33:54 -0700 Subject: [PATCH 15/24] feat(transaction): add the ResetTable action Empties the table -- schema, schema metadata, fragments and indices -- leaving the config, table metadata and base paths alone. This is how a full Overwrite / CREATE OR REPLACE decomposes: reset, then write the fresh schema and data as later actions in the same operation. The id counters keep going across the reset, so a field or fragment added afterwards never reuses an id a stale file might still name. Conflict detection gains its first non-enumerable footprint. A reset writes every coordinate there is, including ones a concurrent set would only mint, so it takes the table exclusively: any concurrent action set is preempted, including a pure append that writes no committed coordinate at all. Two proto tests used ResetTable as their example of a drafted-but- unimplemented action; they now use RefreshRowVersionMetadata. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance-table/src/transaction/action.rs | 8 + .../src/transaction/action/apply.rs | 21 +++ .../src/transaction/action/footprint.rs | 16 ++ .../src/transaction/action/proto.rs | 13 +- .../src/transaction/action/reset_table.rs | 164 ++++++++++++++++++ rust/lance-table/src/transaction/proto.rs | 10 +- 6 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 rust/lance-table/src/transaction/action/reset_table.rs diff --git a/rust/lance-table/src/transaction/action.rs b/rust/lance-table/src/transaction/action.rs index 338ea5bdd37..9e365deb3c4 100644 --- a/rust/lance-table/src/transaction/action.rs +++ b/rust/lance-table/src/transaction/action.rs @@ -44,6 +44,7 @@ mod footprint; mod proto; mod remove_fragment; mod reserve_fragment_ids; +mod reset_table; mod set_deletion_file; mod tombstone_field_data; mod translate; @@ -60,6 +61,7 @@ pub use drop_field::DropField; pub use footprint::{Coordinate, Footprint}; pub use remove_fragment::RemoveFragment; pub use reserve_fragment_ids::ReserveFragmentIds; +pub use reset_table::ResetTable; pub use set_deletion_file::SetDeletionFile; pub use tombstone_field_data::TombstoneFieldData; @@ -171,6 +173,7 @@ pub enum Action { AlterField(AlterField), DropField(DropField), ReserveFragmentIds(ReserveFragmentIds), + ResetTable(ResetTable), } impl Action { @@ -186,6 +189,7 @@ impl Action { Self::AlterField(_) => "AlterField", Self::DropField(_) => "DropField", Self::ReserveFragmentIds(_) => "ReserveFragmentIds", + Self::ResetTable(_) => "ResetTable", } } @@ -204,6 +208,8 @@ impl Action { // Dropping a field discards the values it held. Self::DropField(_) => true, // Other schema and base-path changes touch no row values. + // Emptying the table discards every row it held. + Self::ResetTable(_) => true, // Reserving ids writes no rows either. Self::AddField(_) | Self::AddBase(_) @@ -225,6 +231,7 @@ impl Action { Self::AlterField(action) => action.apply(state), Self::DropField(action) => action.apply(state), Self::ReserveFragmentIds(action) => action.apply(state), + Self::ResetTable(action) => action.apply(state), } } @@ -241,6 +248,7 @@ impl Action { Self::AlterField(action) => action.footprint(footprint), Self::DropField(action) => action.footprint(footprint), Self::ReserveFragmentIds(action) => action.footprint(footprint), + Self::ResetTable(action) => action.footprint(footprint), } } } diff --git a/rust/lance-table/src/transaction/action/apply.rs b/rust/lance-table/src/transaction/action/apply.rs index 5eb79d4e954..a03302bc05b 100644 --- a/rust/lance-table/src/transaction/action/apply.rs +++ b/rust/lance-table/src/transaction/action/apply.rs @@ -69,10 +69,14 @@ impl Transaction { new_bases, rebound_fields, reserved_fragment_ids, + reset, .. } = state; let mut indices = current_indices; + if reset { + indices.clear(); + } prune_rebound_fields_from_indices(&mut indices, &rebound_fields); Self::retain_relevant_indices(&mut indices, &schema, &fragments); @@ -147,6 +151,10 @@ pub(super) struct ApplyState { /// Fields whose backing data changed, per fragment. An index covering such /// a field no longer describes that fragment's contents. rebound_fields: HashMap>, + + /// Whether the table was reset, which discards every index outright rather + /// than pruning fragments out of them. + reset: bool, } impl ApplyState { @@ -170,6 +178,7 @@ impl ApplyState { minted_fragments: HashSet::new(), reserved_fragment_ids: None, rebound_fields: HashMap::new(), + reset: false, } } @@ -213,6 +222,18 @@ impl ApplyState { true } + /// Empty the table: no schema, no fragments, no indices. The id counters + /// keep going, so a field or fragment added afterwards never reuses an id an + /// old file might still name. + pub(super) fn reset(&mut self) { + self.schema.fields.clear(); + self.schema.metadata.clear(); + self.fragments.clear(); + self.minted_fragments.clear(); + self.rebound_fields.clear(); + self.reset = true; + } + /// The base paths this apply can see: the read version's, plus the ones /// earlier actions in this operation minted. pub(super) fn bases(&self) -> impl Iterator { diff --git a/rust/lance-table/src/transaction/action/footprint.rs b/rust/lance-table/src/transaction/action/footprint.rs index 822a8ecfd08..fe4d76f6626 100644 --- a/rust/lance-table/src/transaction/action/footprint.rs +++ b/rust/lance-table/src/transaction/action/footprint.rs @@ -80,10 +80,20 @@ pub struct Footprint { /// is therefore not caught here and fails when it is applied against the /// version where the child no longer exists. removed_fields: HashSet, + /// Whether this set rewrites the table wholesale. Such a set writes every + /// coordinate there is, including ones a concurrent set would only mint, so + /// it is tracked as a flag rather than enumerated. + exclusive: bool, } impl Footprint { pub fn conflicts_with(&self, other: &Self) -> bool { + // A wholesale rewrite leaves nothing for a concurrent set to land on -- + // not even an append, whose rows the reset would discard or resurrect + // depending on which commit won. + if self.exclusive || other.exclusive { + return true; + } if !self.writes.is_disjoint(&other.writes) { return true; } @@ -122,6 +132,12 @@ impl Footprint { self.removed_fragments.insert(fragment); } + /// Mark this set as rewriting the whole table, conflicting with any + /// concurrent set whatsoever. + pub(super) fn take_exclusive(&mut self) { + self.exclusive = true; + } + pub(super) fn remove_field(&mut self, field: i32) { self.add(Coordinate::FieldDefinition(field)); self.removed_fields.insert(field); diff --git a/rust/lance-table/src/transaction/action/proto.rs b/rust/lance-table/src/transaction/action/proto.rs index 9868d0f532c..54a736d3591 100644 --- a/rust/lance-table/src/transaction/action/proto.rs +++ b/rust/lance-table/src/transaction/action/proto.rs @@ -133,6 +133,7 @@ impl From<&Action> for pb::Action { Action::ReserveFragmentIds(action) => { pb::action::Action::ReserveFragmentIds(action.into()) } + Action::ResetTable(action) => pb::action::Action::ResetTable(action.into()), }; Self { action: Some(action), @@ -169,6 +170,9 @@ impl TryFrom for Action { Some(pb::action::Action::ReserveFragmentIds(action)) => { Ok(Self::ReserveFragmentIds(action.try_into()?)) } + Some(pb::action::Action::ResetTable(action)) => { + Ok(Self::ResetTable(action.try_into()?)) + } // The drafted vocabulary is larger than what is implemented. Reject // rather than skip: silently dropping an action would apply a // partial transaction. @@ -190,7 +194,7 @@ mod tests { use crate::rowids::version::RowDatasetVersionMeta; use crate::transaction::action::{ AddBase, AddDataFile, AddField, AddFragment, AlterField, DropField, RemoveFragment, - ReserveFragmentIds, SetDeletionFile, TombstoneFieldData, + ReserveFragmentIds, ResetTable, SetDeletionFile, TombstoneFieldData, }; use arrow_schema::{DataType, Field as ArrowField}; use lance_core::datatypes::Field; @@ -255,6 +259,7 @@ mod tests { }), Action::DropField(DropField { field: 3 }), Action::ReserveFragmentIds(ReserveFragmentIds { count: 4 }), + Action::ResetTable(ResetTable), ] } @@ -289,7 +294,11 @@ mod tests { #[test] fn test_unimplemented_action_is_rejected() { let message = pb::Action { - action: Some(pb::action::Action::ResetTable(pb::ResetTable {})), + action: Some(pb::action::Action::RefreshRowVersionMetadata( + pb::RefreshRowVersionMetadata { + fragment_ids: vec![1], + }, + )), }; let error = Action::try_from(message).unwrap_err(); assert!( diff --git a/rust/lance-table/src/transaction/action/reset_table.rs b/rust/lance-table/src/transaction/action/reset_table.rs new file mode 100644 index 00000000000..516e137c5ed --- /dev/null +++ b/rust/lance-table/src/transaction/action/reset_table.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Reset the table to an empty state. + +use super::Footprint; +use super::apply::ApplyState; +use crate::format::pb; +use lance_core::Result; +use lance_core::deepsize::DeepSizeOf; + +/// Reset the table to an empty state, in preparation for a fresh schema and +/// data written by later actions in the same operation. +/// +/// This is how a full `Overwrite` / `CREATE OR REPLACE` decomposes. It drops the +/// entire schema, the schema metadata, all fragments and all indices, and +/// preserves the table config, the table metadata and the base paths -- change +/// those with a [`ConfigUpdate`](super::ConfigUpdate) or an +/// [`AddBase`](super::AddBase) in the same operation. +/// +/// The id counters are not reset. A field or fragment added after the reset gets +/// a fresh id, so a stale file naming an old id can never be mistaken for the +/// new table's data. +#[derive(Debug, Clone, PartialEq, DeepSizeOf, Default)] +pub struct ResetTable; + +impl ResetTable { + pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { + state.reset(); + Ok(()) + } + + /// Everything. A reset writes every coordinate there is, including ones a + /// concurrent set would only mint, so it takes the table exclusively rather + /// than enumerating them. + pub(super) fn footprint(&self, footprint: &mut Footprint) { + footprint.take_exclusive(); + } +} + +impl From<&ResetTable> for pb::ResetTable { + fn from(_value: &ResetTable) -> Self { + Self {} + } +} + +impl TryFrom for ResetTable { + type Error = lance_core::Error; + + fn try_from(_message: pb::ResetTable) -> Result { + Ok(Self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::DataFile; + use crate::transaction::action::test_support::{ + added_field, apply, apply_with_indices, backed_manifest, + }; + use crate::transaction::action::{Action, AddDataFile, AddField, AddFragment, Ref}; + use crate::transaction::test_support::sample_index_metadata; + + fn reset() -> Action { + Action::ResetTable(ResetTable) + } + + #[test] + fn test_reset_empties_the_table() { + let manifest = backed_manifest(); + assert!(!manifest.fragments.is_empty()); + assert!(!manifest.schema.fields.is_empty()); + + let (next, indices) = + apply_with_indices(&manifest, vec![reset()], vec![sample_index_metadata("idx")]) + .unwrap(); + + assert!(next.fragments.is_empty()); + assert!(next.schema.fields.is_empty()); + assert!(indices.is_empty()); + } + + #[test] + fn test_reset_preserves_config_and_base_paths() { + let mut manifest = backed_manifest(); + manifest.config.insert("lance.keep".into(), "yes".into()); + manifest.table_metadata.insert("owner".into(), "me".into()); + + let next = apply(&manifest, vec![reset()]).unwrap(); + + assert_eq!(next.config.get("lance.keep"), Some(&"yes".to_string())); + assert_eq!(next.table_metadata.get("owner"), Some(&"me".to_string())); + } + + #[test] + fn test_reset_then_rebuild_in_one_operation() { + let next = apply( + &backed_manifest(), + vec![ + reset(), + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("fresh"), + }), + Action::AddFragment(AddFragment { + local: 0, + physical_rows: 4, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Local(0), + file: DataFile::new_unstarted("data/fresh.lance", 2, 0), + field_ids: vec![Ref::Local(0)], + data_change: true, + }), + ], + ) + .unwrap(); + + // The rebuilt table holds only what the operation wrote after the reset, + // and its ids continue past the ones the old table used. + assert_eq!(next.schema.fields.len(), 1); + let field = next.schema.field("fresh").unwrap(); + assert_ne!(field.id, 0, "a fresh field must not reuse a dropped id"); + assert_eq!(next.fragments.len(), 1); + assert_ne!(next.fragments[0].id, 0); + assert_eq!(next.fragments[0].files[0].fields.as_ref(), &[field.id]); + } + + #[test] + fn test_reset_conflicts_with_everything() { + use crate::transaction::action::{Footprint, UserAction, UserOperation}; + + let footprint = |actions| { + Footprint::from(&UserOperation::new( + "test", + vec![UserAction::new("step", actions)], + )) + }; + + let reset = footprint(vec![reset()]); + // Even a pure append, which writes no committed coordinate at all, is + // preempted: its rows would either vanish or survive the reset + // depending on which commit landed first. + let append = footprint(vec![Action::AddFragment(AddFragment { + local: 0, + physical_rows: 4, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + })]); + + assert!(reset.conflicts_with(&append)); + assert!(append.conflicts_with(&reset)); + assert!(reset.conflicts_with(&reset.clone())); + assert!(!append.conflicts_with(&append.clone())); + } +} diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs index 03148ffd1e6..306188b1fb6 100644 --- a/rust/lance-table/src/transaction/proto.rs +++ b/rust/lance-table/src/transaction/proto.rs @@ -938,13 +938,17 @@ mod tests { uuid: Uuid::new_v4().to_string(), operation: Some(pb::transaction::Operation::UserOperation( pb::UserOperation { - description: "DROP TABLE t".to_string(), + description: "MERGE INTO t".to_string(), uuid: Uuid::new_v4().to_string(), read_version: 1, actions: vec![pb::UserAction { - description: "reset".to_string(), + description: "refresh row versions".to_string(), actions: vec![pb::Action { - action: Some(pb::action::Action::ResetTable(pb::ResetTable {})), + action: Some(pb::action::Action::RefreshRowVersionMetadata( + pb::RefreshRowVersionMetadata { + fragment_ids: vec![1], + }, + )), }], }], }, From d4eb9e1f6ece48868836715ef7546eb73cb2a53c Mon Sep 17 00:00:00 2001 From: Will Jones Date: Mon, 17 Aug 2026 14:43:25 -0700 Subject: [PATCH 16/24] feat(transaction): add the ConfigUpdate action Edits the four string maps a manifest carries -- dataset config, table metadata, schema metadata and per-field metadata -- with the same UpdateMap the legacy UpdateConfig operation takes. Per-field updates are keyed by Ref, so a field minted earlier in the same operation can be given metadata before it has a committed id. The unenforced primary and clustering keys keep the immutability rules the legacy path enforces: each is rejected if changed once set, or if a reserved key is written with a value that installs no valid key. The check runs on every apply including a conflict rebase, so it catches the concurrent-writer race too. Conflicts are per key: two operations editing different keys of the same map commute. A replacement writes keys it does not name, so like a fragment removal it is matched by map rather than by key, and two replacements of one map collide even when both are clears. A field's metadata belongs to the field, so dropping the field collides with a concurrent update to its metadata. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance-table/src/transaction/action.rs | 11 +- .../src/transaction/action/apply.rs | 20 + .../src/transaction/action/config_update.rs | 488 ++++++++++++++++++ .../src/transaction/action/footprint.rs | 68 ++- .../src/transaction/action/proto.rs | 28 +- 5 files changed, 608 insertions(+), 7 deletions(-) create mode 100644 rust/lance-table/src/transaction/action/config_update.rs diff --git a/rust/lance-table/src/transaction/action.rs b/rust/lance-table/src/transaction/action.rs index 9e365deb3c4..c5a80eefb9b 100644 --- a/rust/lance-table/src/transaction/action.rs +++ b/rust/lance-table/src/transaction/action.rs @@ -39,6 +39,7 @@ mod add_field; mod add_fragment; mod alter_field; mod apply; +mod config_update; mod drop_field; mod footprint; mod proto; @@ -57,8 +58,9 @@ pub use add_data_file::AddDataFile; pub use add_field::AddField; pub use add_fragment::AddFragment; pub use alter_field::AlterField; +pub use config_update::{ConfigUpdate, FieldMetadataUpdate}; pub use drop_field::DropField; -pub use footprint::{Coordinate, Footprint}; +pub use footprint::{ConfigMap, Coordinate, Footprint}; pub use remove_fragment::RemoveFragment; pub use reserve_fragment_ids::ReserveFragmentIds; pub use reset_table::ResetTable; @@ -174,6 +176,7 @@ pub enum Action { DropField(DropField), ReserveFragmentIds(ReserveFragmentIds), ResetTable(ResetTable), + ConfigUpdate(ConfigUpdate), } impl Action { @@ -190,6 +193,7 @@ impl Action { Self::DropField(_) => "DropField", Self::ReserveFragmentIds(_) => "ReserveFragmentIds", Self::ResetTable(_) => "ResetTable", + Self::ConfigUpdate(_) => "ConfigUpdate", } } @@ -214,7 +218,8 @@ impl Action { Self::AddField(_) | Self::AddBase(_) | Self::AlterField(_) - | Self::ReserveFragmentIds(_) => false, + | Self::ReserveFragmentIds(_) + | Self::ConfigUpdate(_) => false, } } @@ -232,6 +237,7 @@ impl Action { Self::DropField(action) => action.apply(state), Self::ReserveFragmentIds(action) => action.apply(state), Self::ResetTable(action) => action.apply(state), + Self::ConfigUpdate(action) => action.apply(state), } } @@ -249,6 +255,7 @@ impl Action { Self::DropField(action) => action.footprint(footprint), Self::ReserveFragmentIds(action) => action.footprint(footprint), Self::ResetTable(action) => action.footprint(footprint), + Self::ConfigUpdate(action) => action.footprint(footprint), } } } diff --git a/rust/lance-table/src/transaction/action/apply.rs b/rust/lance-table/src/transaction/action/apply.rs index a03302bc05b..d51ce3c87b7 100644 --- a/rust/lance-table/src/transaction/action/apply.rs +++ b/rust/lance-table/src/transaction/action/apply.rs @@ -70,6 +70,8 @@ impl Transaction { rebound_fields, reserved_fragment_ids, reset, + config: dataset_config, + table_metadata, .. } = state; @@ -94,6 +96,9 @@ impl Transaction { manifest.base_paths.insert(base.id, base); } + manifest.config = dataset_config; + manifest.table_metadata = table_metadata; + // A reserved id backs no fragment, so the manifest assembly cannot // derive it from the fragment list; raise the high-water mark to cover // the range so a later writer's ids are not handed out twice. @@ -130,6 +135,11 @@ pub(super) struct ApplyState { /// base paths, which the manifest assembly inherits from the read version. new_bases: Vec, existing_base_paths: HashMap, + /// The manifest's string maps. Unlike the schema and the fragment list, + /// these are inherited wholesale by the manifest assembly, so an edit has + /// to be written back over the assembled manifest. + config: HashMap, + table_metadata: HashMap, next_fragment_id: u64, next_field_id: i32, @@ -164,6 +174,8 @@ impl ApplyState { fragments: manifest.fragments.as_ref().clone(), new_bases: Vec::new(), existing_base_paths: manifest.base_paths.clone(), + config: manifest.config.clone(), + table_metadata: manifest.table_metadata.clone(), next_fragment_id: manifest.max_fragment_id().map(|id| id + 1).unwrap_or(0), next_field_id: manifest.max_field_id() + 1, next_base_id: manifest @@ -190,6 +202,14 @@ impl ApplyState { &mut self.schema } + pub(super) fn config_mut(&mut self) -> &mut HashMap { + &mut self.config + } + + pub(super) fn table_metadata_mut(&mut self) -> &mut HashMap { + &mut self.table_metadata + } + pub(super) fn fragments_mut(&mut self) -> &mut [Fragment] { &mut self.fragments } diff --git a/rust/lance-table/src/transaction/action/config_update.rs b/rust/lance-table/src/transaction/action/config_update.rs new file mode 100644 index 00000000000..f68556cad7b --- /dev/null +++ b/rust/lance-table/src/transaction/action/config_update.rs @@ -0,0 +1,488 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Apply config and metadata updates. + +use super::apply::ApplyState; +use super::proto::required; +use super::{ConfigMap, Footprint, Ref}; +use crate::format::pb; +use crate::transaction::UpdateMap; +use crate::transaction::update_map::apply_update_map; +use lance_core::datatypes::{ + Field, LANCE_UNENFORCED_CLUSTERING_KEY_POSITION, LANCE_UNENFORCED_PRIMARY_KEY, + LANCE_UNENFORCED_PRIMARY_KEY_POSITION, +}; +use lance_core::deepsize::DeepSizeOf; +use lance_core::{Error, Result}; + +/// Apply config and metadata updates. +/// +/// Each of the four string maps a manifest carries is edited by an +/// [`UpdateMap`]: a list of keys to set or delete, or a wholesale replacement. +/// Absent means "leave this map alone", which is why every field is optional. +/// +/// This is reference-stable rather than a delta -- config keys and field ids are +/// stable coordinates -- so two operations editing different keys commute. +#[derive(Debug, Clone, PartialEq, DeepSizeOf, Default)] +pub struct ConfigUpdate { + /// Dataset config. + pub config: Option, + /// Table metadata. + pub table_metadata: Option, + /// Schema-level metadata. + pub schema_metadata: Option, + /// Per-field metadata, in application order. Keyed by [`Ref`] so a field + /// minted earlier in the same operation can be given metadata. + pub field_metadata: Vec, +} + +/// One field's metadata within a [`ConfigUpdate`]. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct FieldMetadataUpdate { + pub field: Ref, + pub updates: UpdateMap, +} + +impl ConfigUpdate { + pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { + if let Some(updates) = &self.config { + apply_update_map(state.config_mut(), updates); + } + if let Some(updates) = &self.table_metadata { + apply_update_map(state.table_metadata_mut(), updates); + } + if let Some(updates) = &self.schema_metadata { + apply_update_map(&mut state.schema_mut().metadata, updates); + } + if self.field_metadata.is_empty() { + return Ok(()); + } + + // The unenforced primary and clustering keys are reserved schema + // properties: each is immutable once set, and its reserved keys cannot + // be written with an invalid value. Capture what they were before the + // updates land so a violation can be rejected below. This runs on every + // apply, including a conflict rebase, so it also catches the + // concurrent-writer race. + let primary_key_before = unenforced_primary_key(state); + let clustering_key_before = unenforced_clustering_key(state); + let writes_primary_key = self.writes_any(&[ + LANCE_UNENFORCED_PRIMARY_KEY, + LANCE_UNENFORCED_PRIMARY_KEY_POSITION, + ]); + let writes_clustering_key = self.writes_any(&[LANCE_UNENFORCED_CLUSTERING_KEY_POSITION]); + + for update in &self.field_metadata { + let field_id = state.resolve_field(update.field)?; + let field = state + .schema_mut() + .field_by_id_mut(field_id) + .ok_or_else(|| { + Error::invalid_input(format!( + "ConfigUpdate names field {field_id}, which does not exist" + )) + })?; + apply_update_map(&mut field.metadata, &update.updates); + refresh_reserved_key_positions(field); + } + + reject_reserved_key_change( + "primary", + &primary_key_before, + &unenforced_primary_key(state), + writes_primary_key, + )?; + reject_reserved_key_change( + "clustering", + &clustering_key_before, + &unenforced_clustering_key(state), + writes_clustering_key, + ) + } + + /// The keys this update names, or the whole map when it replaces one. A + /// field's metadata belongs to the field, so dropping the field also + /// collides with an update to it. + pub(super) fn footprint(&self, footprint: &mut Footprint) { + for (map, update) in [ + (ConfigMap::Config, &self.config), + (ConfigMap::TableMetadata, &self.table_metadata), + (ConfigMap::SchemaMetadata, &self.schema_metadata), + ] { + if let Some(update) = update { + footprint.add_map_update(map, update); + } + } + for update in &self.field_metadata { + // A field minted in this operation has no committed id, so no + // concurrent writer can be naming it. + if let Some(id) = update.field.committed() + && let Ok(id) = i32::try_from(id) + { + footprint.add_map_update(ConfigMap::Field(id), &update.updates); + } + } + } + + fn writes_any(&self, keys: &[&str]) -> bool { + self.field_metadata.iter().any(|update| { + update + .updates + .update_entries + .iter() + .any(|entry| keys.contains(&entry.key.as_str())) + }) + } +} + +fn unenforced_primary_key(state: &ApplyState) -> Vec { + state + .schema() + .unenforced_primary_key() + .iter() + .map(|field| field.id) + .collect() +} + +fn unenforced_clustering_key(state: &ApplyState) -> Vec { + state + .schema() + .unenforced_clustering_key() + .iter() + .map(|field| field.id) + .collect() +} + +/// A field caches its reserved-key positions alongside the metadata they are +/// parsed from, so the cache has to be rebuilt whenever the metadata changes. +fn refresh_reserved_key_positions(field: &mut Field) { + field.unenforced_primary_key_position = field + .metadata + .get(LANCE_UNENFORCED_PRIMARY_KEY_POSITION) + .and_then(|value| value.parse::().ok()) + .or_else(|| { + field + .metadata + .get(LANCE_UNENFORCED_PRIMARY_KEY) + .filter(|value| matches!(value.to_lowercase().as_str(), "true" | "1" | "yes")) + .map(|_| 0) + }); + field.unenforced_clustering_key_position = field + .metadata + .get(LANCE_UNENFORCED_CLUSTERING_KEY_POSITION) + .and_then(|value| value.parse::().ok()); +} + +fn reject_reserved_key_change( + which: &str, + before: &[i32], + after: &[i32], + writes_reserved_key: bool, +) -> Result<()> { + if !before.is_empty() { + if writes_reserved_key || after != before { + return Err(Error::invalid_input(format!( + "the unenforced {which} key is a reserved key and cannot be changed once set" + ))); + } + } else if writes_reserved_key && after.is_empty() { + // A reserved key was written but installed no valid key, e.g. a + // non-marker flag value or a non-numeric position. + return Err(Error::invalid_input(format!( + "the unenforced {which} key is a reserved key and cannot be set to an invalid value" + ))); + } + Ok(()) +} + +impl From<&ConfigUpdate> for pb::ConfigUpdate { + fn from(value: &ConfigUpdate) -> Self { + Self { + config: value.config.as_ref().map(pb::UpdateMap::from), + table_metadata: value.table_metadata.as_ref().map(pb::UpdateMap::from), + schema_metadata: value.schema_metadata.as_ref().map(pb::UpdateMap::from), + field_metadata: value + .field_metadata + .iter() + .map(|update| pb::config_update::FieldMetadata { + field: Some(update.field.into()), + updates: Some(pb::UpdateMap::from(&update.updates)), + }) + .collect(), + } + } +} + +impl TryFrom for ConfigUpdate { + type Error = Error; + + fn try_from(message: pb::ConfigUpdate) -> Result { + Ok(Self { + config: message.config.as_ref().map(UpdateMap::from), + table_metadata: message.table_metadata.as_ref().map(UpdateMap::from), + schema_metadata: message.schema_metadata.as_ref().map(UpdateMap::from), + field_metadata: message + .field_metadata + .into_iter() + .map(|update| { + Ok(FieldMetadataUpdate { + field: required(update.field, "ConfigUpdate.field_metadata.field")? + .try_into()?, + updates: UpdateMap::from(&required( + update.updates, + "ConfigUpdate.field_metadata.updates", + )?), + }) + }) + .collect::>>()?, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::transaction::action::test_support::{added_field, apply, backed_manifest}; + use crate::transaction::action::{Action, AddField, Footprint, UserAction, UserOperation}; + use crate::transaction::update_map::UpdateMapEntry; + + fn merge(entries: &[(&str, Option<&str>)]) -> UpdateMap { + UpdateMap { + update_entries: entries.iter().map(|entry| (*entry).into()).collect(), + replace: false, + } + } + + fn replace(entries: &[(&str, &str)]) -> UpdateMap { + UpdateMap { + update_entries: entries + .iter() + .map(|(key, value)| UpdateMapEntry { + key: (*key).into(), + value: Some((*value).into()), + }) + .collect(), + replace: true, + } + } + + fn footprint(actions: Vec) -> Footprint { + Footprint::from(&UserOperation::new( + "test", + vec![UserAction::new("step", actions)], + )) + } + + #[test] + fn test_config_update_merges_and_deletes() { + let mut manifest = backed_manifest(); + manifest.config.insert("keep".into(), "yes".into()); + manifest.config.insert("drop".into(), "yes".into()); + + let next = apply( + &manifest, + vec![Action::ConfigUpdate(ConfigUpdate { + config: Some(merge(&[("drop", None), ("added", Some("1"))])), + ..Default::default() + })], + ) + .unwrap(); + + assert_eq!(next.config.get("keep"), Some(&"yes".to_string())); + assert_eq!(next.config.get("added"), Some(&"1".to_string())); + assert!(!next.config.contains_key("drop")); + } + + #[test] + fn test_config_update_replaces_a_whole_map() { + let mut manifest = backed_manifest(); + manifest.table_metadata.insert("old".into(), "yes".into()); + + let next = apply( + &manifest, + vec![Action::ConfigUpdate(ConfigUpdate { + table_metadata: Some(replace(&[("new", "1")])), + ..Default::default() + })], + ) + .unwrap(); + + assert!(!next.table_metadata.contains_key("old")); + assert_eq!(next.table_metadata.get("new"), Some(&"1".to_string())); + } + + #[test] + fn test_config_update_edits_schema_and_field_metadata() { + let next = apply( + &backed_manifest(), + vec![Action::ConfigUpdate(ConfigUpdate { + schema_metadata: Some(merge(&[("schema", Some("yes"))])), + field_metadata: vec![FieldMetadataUpdate { + field: Ref::Committed(0), + updates: merge(&[("comment", Some("the id"))]), + }], + ..Default::default() + })], + ) + .unwrap(); + + assert_eq!(next.schema.metadata.get("schema"), Some(&"yes".to_string())); + let field = next.schema.field_by_id(0).unwrap(); + assert_eq!(field.metadata.get("comment"), Some(&"the id".to_string())); + } + + #[test] + fn test_config_update_can_name_a_field_minted_in_the_same_operation() { + let next = apply( + &backed_manifest(), + vec![ + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("fresh"), + }), + Action::ConfigUpdate(ConfigUpdate { + field_metadata: vec![FieldMetadataUpdate { + field: Ref::Local(0), + updates: merge(&[("comment", Some("brand new"))]), + }], + ..Default::default() + }), + ], + ) + .unwrap(); + + let field = next.schema.field("fresh").unwrap(); + assert_eq!( + field.metadata.get("comment"), + Some(&"brand new".to_string()) + ); + } + + #[test] + fn test_config_update_rejects_a_missing_field() { + let error = apply( + &backed_manifest(), + vec![Action::ConfigUpdate(ConfigUpdate { + field_metadata: vec![FieldMetadataUpdate { + field: Ref::Committed(7), + updates: merge(&[("comment", Some("nope"))]), + }], + ..Default::default() + })], + ) + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + assert!(error.to_string().contains("field 7"), "{error}"); + } + + #[test] + fn test_config_update_sets_then_refuses_to_change_the_primary_key() { + let set_key = |field: u64| { + Action::ConfigUpdate(ConfigUpdate { + field_metadata: vec![FieldMetadataUpdate { + field: Ref::Committed(field), + updates: merge(&[(LANCE_UNENFORCED_PRIMARY_KEY, Some("true"))]), + }], + ..Default::default() + }) + }; + + let next = apply(&backed_manifest(), vec![set_key(0)]).unwrap(); + assert_eq!( + next.schema + .unenforced_primary_key() + .iter() + .map(|field| field.id) + .collect::>(), + vec![0] + ); + + let error = apply(&next, vec![set_key(0)]).unwrap_err(); + assert!( + error.to_string().contains("cannot be changed once set"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_config_update_rejects_an_invalid_primary_key() { + let error = apply( + &backed_manifest(), + vec![Action::ConfigUpdate(ConfigUpdate { + field_metadata: vec![FieldMetadataUpdate { + field: Ref::Committed(0), + updates: merge(&[( + LANCE_UNENFORCED_PRIMARY_KEY_POSITION, + Some("not a number"), + )]), + }], + ..Default::default() + })], + ) + .unwrap_err(); + + assert!( + error + .to_string() + .contains("cannot be set to an invalid value"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_edits_to_different_keys_commute() { + let update = |key: &str| { + vec![Action::ConfigUpdate(ConfigUpdate { + config: Some(merge(&[(key, Some("1"))])), + ..Default::default() + })] + }; + + assert!(!footprint(update("a")).conflicts_with(&footprint(update("b")))); + assert!(footprint(update("a")).conflicts_with(&footprint(update("a")))); + } + + #[test] + fn test_a_replacement_collides_with_any_edit_to_the_same_map() { + let replaced = footprint(vec![Action::ConfigUpdate(ConfigUpdate { + config: Some(replace(&[("a", "1")])), + ..Default::default() + })]); + let merged = footprint(vec![Action::ConfigUpdate(ConfigUpdate { + config: Some(merge(&[("untouched-by-name", Some("1"))])), + ..Default::default() + })]); + let other_map = footprint(vec![Action::ConfigUpdate(ConfigUpdate { + table_metadata: Some(merge(&[("a", Some("1"))])), + ..Default::default() + })]); + + assert!(replaced.conflicts_with(&merged)); + // Two clears of the same map name no key at all, and still collide. + assert!(replaced.conflicts_with(&replaced.clone())); + // A different map is a different coordinate space. + assert!(!replaced.conflicts_with(&other_map)); + } + + #[test] + fn test_dropping_a_field_collides_with_updating_its_metadata() { + use crate::transaction::action::DropField; + + let metadata = footprint(vec![Action::ConfigUpdate(ConfigUpdate { + field_metadata: vec![FieldMetadataUpdate { + field: Ref::Committed(1), + updates: merge(&[("comment", Some("x"))]), + }], + ..Default::default() + })]); + let dropped = footprint(vec![Action::DropField(DropField { field: 1 })]); + let other = footprint(vec![Action::DropField(DropField { field: 2 })]); + + assert!(dropped.conflicts_with(&metadata)); + assert!(metadata.conflicts_with(&dropped)); + assert!(!other.conflicts_with(&metadata)); + } +} diff --git a/rust/lance-table/src/transaction/action/footprint.rs b/rust/lance-table/src/transaction/action/footprint.rs index fe4d76f6626..7a751d5cbe7 100644 --- a/rust/lance-table/src/transaction/action/footprint.rs +++ b/rust/lance-table/src/transaction/action/footprint.rs @@ -17,6 +17,7 @@ //! module. This module holds the coordinate space and the comparison. use super::{Ref, UserOperation}; +use crate::transaction::UpdateMap; use std::collections::HashSet; /// One thing an action set writes. @@ -38,6 +39,21 @@ pub enum Coordinate { BaseName(Option), /// A base path's location, which the manifest requires to be unique. BaseLocation(String), + /// One key in one of the manifest's string maps. + ConfigEntry { map: ConfigMap, key: String }, +} + +/// One of the string maps a manifest carries. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum ConfigMap { + /// Dataset config. + Config, + /// Table metadata. + TableMetadata, + /// Schema-level metadata. + SchemaMetadata, + /// One field's metadata, by field id. + Field(i32), } impl Coordinate { @@ -46,7 +62,10 @@ impl Coordinate { match self { Self::FragmentExistence(id) | Self::FragmentDeletions(id) => Some(*id), Self::FieldData { fragment, .. } => Some(*fragment), - Self::FieldDefinition(_) | Self::BaseName(_) | Self::BaseLocation(_) => None, + Self::FieldDefinition(_) + | Self::BaseName(_) + | Self::BaseLocation(_) + | Self::ConfigEntry { .. } => None, } } @@ -55,10 +74,25 @@ impl Coordinate { match self { Self::FieldData { field, .. } => Some(*field), Self::FieldDefinition(id) => Some(*id), + // A field's metadata goes with the field, so dropping the field + // writes over a concurrent update to its metadata. + Self::ConfigEntry { + map: ConfigMap::Field(id), + .. + } => Some(*id), Self::FragmentExistence(_) | Self::FragmentDeletions(_) | Self::BaseName(_) - | Self::BaseLocation(_) => None, + | Self::BaseLocation(_) + | Self::ConfigEntry { .. } => None, + } + } + + /// The string map this coordinate is a key in, if it is one. + fn config_map(&self) -> Option<&ConfigMap> { + match self { + Self::ConfigEntry { map, .. } => Some(map), + _ => None, } } } @@ -80,6 +114,10 @@ pub struct Footprint { /// is therefore not caught here and fails when it is applied against the /// version where the child no longer exists. removed_fields: HashSet, + /// String maps this set replaces outright rather than merging into. Like a + /// fragment removal, this writes every key in the map, including keys it + /// does not name, so it is matched by map rather than by key. + replaced_maps: HashSet, /// Whether this set rewrites the table wholesale. Such a set writes every /// coordinate there is, including ones a concurrent set would only mint, so /// it is tracked as a flag rather than enumerated. @@ -97,10 +135,16 @@ impl Footprint { if !self.writes.is_disjoint(&other.writes) { return true; } + // Two sets replacing the same map collide even when neither names a + // key, since clearing a map is a replacement with no entries. + if !self.replaced_maps.is_disjoint(&other.replaced_maps) { + return true; + } self.removes_something_touched_by(other) || other.removes_something_touched_by(self) } - /// Whether this set removes a fragment or field that `other` also writes to. + /// Whether this set wipes out something -- a fragment, a field, a whole + /// string map -- that `other` also writes to. fn removes_something_touched_by(&self, other: &Self) -> bool { other.writes.iter().any(|coordinate| { coordinate @@ -109,6 +153,9 @@ impl Footprint { || coordinate .field() .is_some_and(|id| self.removed_fields.contains(&id)) + || coordinate + .config_map() + .is_some_and(|map| self.replaced_maps.contains(map)) }) } @@ -132,6 +179,21 @@ impl Footprint { self.removed_fragments.insert(fragment); } + /// Record an edit to one of the manifest's string maps: the keys it names, + /// or the whole map when it replaces rather than merges. + pub(super) fn add_map_update(&mut self, map: ConfigMap, update: &UpdateMap) { + if update.replace { + self.replaced_maps.insert(map); + return; + } + for entry in &update.update_entries { + self.add(Coordinate::ConfigEntry { + map: map.clone(), + key: entry.key.clone(), + }); + } + } + /// Mark this set as rewriting the whole table, conflicting with any /// concurrent set whatsoever. pub(super) fn take_exclusive(&mut self) { diff --git a/rust/lance-table/src/transaction/action/proto.rs b/rust/lance-table/src/transaction/action/proto.rs index 54a736d3591..0c0ee4c4cd2 100644 --- a/rust/lance-table/src/transaction/action/proto.rs +++ b/rust/lance-table/src/transaction/action/proto.rs @@ -134,6 +134,7 @@ impl From<&Action> for pb::Action { pb::action::Action::ReserveFragmentIds(action.into()) } Action::ResetTable(action) => pb::action::Action::ResetTable(action.into()), + Action::ConfigUpdate(action) => pb::action::Action::ConfigUpdate(action.into()), }; Self { action: Some(action), @@ -173,6 +174,9 @@ impl TryFrom for Action { Some(pb::action::Action::ResetTable(action)) => { Ok(Self::ResetTable(action.try_into()?)) } + Some(pb::action::Action::ConfigUpdate(action)) => { + Ok(Self::ConfigUpdate(action.try_into()?)) + } // The drafted vocabulary is larger than what is implemented. Reject // rather than skip: silently dropping an action would apply a // partial transaction. @@ -192,9 +196,11 @@ mod tests { use super::*; use crate::format::{BasePath, DataFile, DeletionFile, DeletionFileType, RowIdMeta, pb}; use crate::rowids::version::RowDatasetVersionMeta; + use crate::transaction::UpdateMap; use crate::transaction::action::{ - AddBase, AddDataFile, AddField, AddFragment, AlterField, DropField, RemoveFragment, - ReserveFragmentIds, ResetTable, SetDeletionFile, TombstoneFieldData, + AddBase, AddDataFile, AddField, AddFragment, AlterField, ConfigUpdate, DropField, + FieldMetadataUpdate, RemoveFragment, ReserveFragmentIds, ResetTable, SetDeletionFile, + TombstoneFieldData, }; use arrow_schema::{DataType, Field as ArrowField}; use lance_core::datatypes::Field; @@ -260,6 +266,24 @@ mod tests { Action::DropField(DropField { field: 3 }), Action::ReserveFragmentIds(ReserveFragmentIds { count: 4 }), Action::ResetTable(ResetTable), + Action::ConfigUpdate(ConfigUpdate { + config: Some(UpdateMap { + update_entries: vec![("a", "1").into(), ("b", None).into()], + replace: false, + }), + table_metadata: None, + schema_metadata: Some(UpdateMap { + update_entries: vec![("c", "2").into()], + replace: true, + }), + field_metadata: vec![FieldMetadataUpdate { + field: Ref::Local(3), + updates: UpdateMap { + update_entries: vec![("d", "3").into()], + replace: false, + }, + }], + }), ] } From 9e814fea742a2a694a8a19010c76d8824642e1dd Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 18 Aug 2026 12:57:41 -0700 Subject: [PATCH 17/24] refactor(transaction): rename UserOperation to CompositeOperation `UserOperation` read as a category next to `Operation::Append` rather than naming what distinguishes it. Rename it to `CompositeOperation`, which says the thing: a composite of granular actions committed atomically. `UserAction` keeps its name -- it is the user-facing grouping of actions, the level a user recognizes, as distinct from the granular actions inside it. Also drops `CompositeOperation::description`. Nothing reads it, and every construction site set it to something the step descriptions already imply. The per-step `UserAction::description` is what keeps history readable; an operation-level name can come back if squashing turns out to need one. Co-Authored-By: Claude Opus 5 (1M context) --- protos/transaction/actions.proto | 22 +++--- protos/transaction/transaction.proto | 2 +- rust/lance-table/src/transaction/action.rs | 58 +++++++--------- .../src/transaction/action/apply.rs | 8 +-- .../src/transaction/action/config_update.rs | 9 ++- .../src/transaction/action/footprint.rs | 15 ++--- .../src/transaction/action/proto.rs | 31 ++++----- .../src/transaction/action/reset_table.rs | 9 ++- .../src/transaction/action/test_support.rs | 9 ++- .../src/transaction/action/translate.rs | 6 +- rust/lance-table/src/transaction/conflicts.rs | 4 +- .../src/transaction/manifest_build.rs | 6 +- rust/lance-table/src/transaction/operation.rs | 8 +-- rust/lance-table/src/transaction/proto.rs | 36 +++++----- rust/lance/src/io/commit/conflict_resolver.rs | 67 ++++++++++++------- rust/lance/tests/composite_transaction.rs | 46 ++++++------- 16 files changed, 164 insertions(+), 172 deletions(-) diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto index 949d8c95ef1..823c5d840d5 100644 --- a/protos/transaction/actions.proto +++ b/protos/transaction/actions.proto @@ -13,7 +13,7 @@ package lance.table; /* * Action-based transactions (Transaction V2) — DRAFT. * - * A `UserOperation` replaces the single legacy `Operation` (see + * A `CompositeOperation` replaces the single legacy `Operation` (see * transaction.proto) with an ordered list of granular `Action`s that commit * atomically as one manifest change. This file is the wire draft only. * @@ -21,7 +21,7 @@ package lance.table; * ------ * The messages and field numbers here are NOT yet a stable contract. Library * support is read-side fail-closed only: a transaction carrying a - * `UserOperation` is rejected on load, and there is no write path. Apply, + * `CompositeOperation` is rejected on load, and there is no write path. Apply, * id translation, and conflict resolution are intentionally absent. * * DESIGN RATIONALE @@ -106,10 +106,10 @@ package lance.table; * id) that may not be committed yet. * * `committed` is an already-assigned id. `local` is a placeholder token minted - * by an `Add*` action earlier in the same `UserOperation`; it resolves to a + * by an `Add*` action earlier in the same `CompositeOperation`; it resolves to a * freshly-allocated committed id at apply, and re-resolves against the target's * counters on merge/rebase. `local` tokens are scoped to a single - * `UserOperation` and must be distinct within it (validated by the writer once + * `CompositeOperation` and must be distinct within it (validated by the writer once * the write path exists). */ message Ref { @@ -123,23 +123,21 @@ message Ref { * A user-facing, composable transaction: an ordered list of user actions that * commit atomically as a single manifest change. */ -message UserOperation { - // Human-readable description, e.g. "INSERT INTO t VALUES (1)". - string description = 1; +message CompositeOperation { // Unique identifier for this operation (matches Transaction.uuid semantics). - string uuid = 2; + string uuid = 1; // The dataset version this operation was planned against. - uint64 read_version = 3; + uint64 read_version = 2; // The ordered list of user actions applied by this operation. - repeated UserAction actions = 4; + repeated UserAction actions = 3; } /* - * A single user-recognizable step within a UserOperation (e.g. "append batch", + * A single user-recognizable step within a CompositeOperation (e.g. "append batch", * "rebuild index"). * * The description keeps the transaction history human-readable. When a range of - * transactions is squashed, each original UserOperation collapses into one + * transactions is squashed, each original CompositeOperation collapses into one * UserAction so the readable sequence survives; the action lists are flattened * when applied to the manifest. */ diff --git a/protos/transaction/transaction.proto b/protos/transaction/transaction.proto index 6da49fd8668..09ba8d41e37 100644 --- a/protos/transaction/transaction.proto +++ b/protos/transaction/transaction.proto @@ -343,7 +343,7 @@ message Transaction { DataOverlay data_overlay = 115; // Action-based transaction (Transaction V2). See actions.proto. // DRAFT: currently rejected on load; no write path. - UserOperation user_operation = 116; + CompositeOperation composite_operation = 116; } // Fields 200/202 (`blob_append` / `blob_overwrite`) previously represented blob dataset ops. diff --git a/rust/lance-table/src/transaction/action.rs b/rust/lance-table/src/transaction/action.rs index c5a80eefb9b..a2f26bef7b9 100644 --- a/rust/lance-table/src/transaction/action.rs +++ b/rust/lance-table/src/transaction/action.rs @@ -4,7 +4,7 @@ //! The action vocabulary of Transaction V2. //! //! Where an [`Operation`](super::Operation) names one whole change and carries a -//! post-image of the parts of the manifest it touches, a [`UserOperation`] is an +//! post-image of the parts of the manifest it touches, a [`CompositeOperation`] is an //! ordered list of [`Action`]s, each recording a single *delta*. Composing //! several changes into one atomic commit and replaying a change onto a //! different version both fall out of that, with no per-operation logic. @@ -30,7 +30,7 @@ //! # Stability //! //! Transaction V2 is a pre-vote draft. Nothing in this module is a compatibility -//! contract, and a transaction carrying a [`UserOperation`] is rejected outright +//! contract, and a transaction carrying a [`CompositeOperation`] is rejected outright //! by libraries that predate it. mod add_base; @@ -76,13 +76,13 @@ use lance_core::deepsize::DeepSizeOf; /// /// [`Ref::Committed`] is a concrete id that already exists in the manifest. /// [`Ref::Local`] is a placeholder token minted by an `Add*` action earlier in -/// the same [`UserOperation`]; it resolves to a freshly-allocated id at apply, +/// the same [`CompositeOperation`]; it resolves to a freshly-allocated id at apply, /// and re-resolves against the target's counters when the operation is replayed /// onto a newer version. That re-resolution is what lets two independent /// `AddField`s on divergent branches become two distinct fields rather than a /// collision. /// -/// Local tokens are scoped to one [`UserOperation`] and must be distinct within +/// Local tokens are scoped to one [`CompositeOperation`] and must be distinct within /// it. The three id spaces do not share a token namespace: a fragment token 0 /// and a field token 0 are unrelated. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, DeepSizeOf)] @@ -116,19 +116,14 @@ impl Ref { /// [`Transaction`](super::Transaction) and are filled in from it, so they are /// not repeated here. #[derive(Debug, Clone, PartialEq, DeepSizeOf, Default)] -pub struct UserOperation { - /// Human-readable description of the whole operation, e.g. `"INSERT INTO t"`. - pub description: String, +pub struct CompositeOperation { /// The ordered steps this operation applies. pub actions: Vec, } -impl UserOperation { - pub fn new(description: impl Into, actions: Vec) -> Self { - Self { - description: description.into(), - actions, - } +impl CompositeOperation { + pub fn new(actions: Vec) -> Self { + Self { actions } } /// Every action in every step, in application order. @@ -137,7 +132,7 @@ impl UserOperation { } } -/// A single user-recognizable step within a [`UserOperation`], e.g. "append +/// A single user-recognizable step within a [`CompositeOperation`], e.g. "append /// batch" or "rebuild index". /// /// The description keeps transaction history readable: when a range of versions @@ -296,25 +291,22 @@ mod tests { #[test] fn test_iter_actions_flattens_steps_in_order() { - let operation = UserOperation::new( - "two steps", - vec![ - UserAction::new( - "first", - vec![Action::RemoveFragment(RemoveFragment { - fragment: Ref::Committed(1), - data_change: true, - })], - ), - UserAction::new( - "second", - vec![Action::RemoveFragment(RemoveFragment { - fragment: Ref::Committed(2), - data_change: true, - })], - ), - ], - ); + let operation = CompositeOperation::new(vec![ + UserAction::new( + "first", + vec![Action::RemoveFragment(RemoveFragment { + fragment: Ref::Committed(1), + data_change: true, + })], + ), + UserAction::new( + "second", + vec![Action::RemoveFragment(RemoveFragment { + fragment: Ref::Committed(2), + data_change: true, + })], + ), + ]); let fragments = operation .iter_actions() .map(|action| match action { diff --git a/rust/lance-table/src/transaction/action/apply.rs b/rust/lance-table/src/transaction/action/apply.rs index d51ce3c87b7..0d080de8a35 100644 --- a/rust/lance-table/src/transaction/action/apply.rs +++ b/rust/lance-table/src/transaction/action/apply.rs @@ -14,7 +14,7 @@ //! modules program against. What each action does with it lives in that action's //! own module. -use super::{Ref, UserOperation}; +use super::{CompositeOperation, Ref}; use crate::format::{BasePath, Fragment, IndexMetadata, Manifest, ManifestBuildConfig}; use crate::rowids::version::build_version_meta; use crate::transaction::Transaction; @@ -34,7 +34,7 @@ impl Transaction { /// when the dataset does not exist yet. pub(in crate::transaction) fn build_manifest_from_actions( &self, - user_operation: &UserOperation, + composite_operation: &CompositeOperation, current_manifest: Option<&Manifest>, current_indices: Vec, transaction_file_path: &str, @@ -54,7 +54,7 @@ impl Transaction { let new_version = current_manifest.version + 1; let mut state = ApplyState::new(current_manifest); - for action in user_operation.iter_actions() { + for action in composite_operation.iter_actions() { action.apply(&mut state)?; } @@ -500,7 +500,7 @@ mod tests { fn test_action_set_cannot_create_a_dataset() { let transaction = Transaction::new( 0, - Operation::UserOperation(UserOperation::new("test", vec![])), + Operation::CompositeOperation(CompositeOperation::new(vec![])), None, ); let error = transaction diff --git a/rust/lance-table/src/transaction/action/config_update.rs b/rust/lance-table/src/transaction/action/config_update.rs index f68556cad7b..544f8c41b16 100644 --- a/rust/lance-table/src/transaction/action/config_update.rs +++ b/rust/lance-table/src/transaction/action/config_update.rs @@ -244,7 +244,7 @@ impl TryFrom for ConfigUpdate { mod tests { use super::*; use crate::transaction::action::test_support::{added_field, apply, backed_manifest}; - use crate::transaction::action::{Action, AddField, Footprint, UserAction, UserOperation}; + use crate::transaction::action::{Action, AddField, CompositeOperation, Footprint, UserAction}; use crate::transaction::update_map::UpdateMapEntry; fn merge(entries: &[(&str, Option<&str>)]) -> UpdateMap { @@ -268,10 +268,9 @@ mod tests { } fn footprint(actions: Vec) -> Footprint { - Footprint::from(&UserOperation::new( - "test", - vec![UserAction::new("step", actions)], - )) + Footprint::from(&CompositeOperation::new(vec![UserAction::new( + "step", actions, + )])) } #[test] diff --git a/rust/lance-table/src/transaction/action/footprint.rs b/rust/lance-table/src/transaction/action/footprint.rs index 7a751d5cbe7..8b8285e2f64 100644 --- a/rust/lance-table/src/transaction/action/footprint.rs +++ b/rust/lance-table/src/transaction/action/footprint.rs @@ -16,7 +16,7 @@ //! Which coordinates an action writes is decided by that action, in its own //! module. This module holds the coordinate space and the comparison. -use super::{Ref, UserOperation}; +use super::{CompositeOperation, Ref}; use crate::transaction::UpdateMap; use std::collections::HashSet; @@ -206,10 +206,10 @@ impl Footprint { } } -impl From<&UserOperation> for Footprint { - fn from(user_operation: &UserOperation) -> Self { +impl From<&CompositeOperation> for Footprint { + fn from(composite_operation: &CompositeOperation) -> Self { let mut footprint = Self::default(); - for action in user_operation.iter_actions() { + for action in composite_operation.iter_actions() { action.footprint(&mut footprint); } footprint @@ -229,10 +229,9 @@ mod tests { use rstest::rstest; fn footprint(actions: Vec) -> Footprint { - Footprint::from(&UserOperation::new( - "test", - vec![UserAction::new("step", actions)], - )) + Footprint::from(&CompositeOperation::new(vec![UserAction::new( + "step", actions, + )])) } fn add_fragment(local: u32) -> Action { diff --git a/rust/lance-table/src/transaction/action/proto.rs b/rust/lance-table/src/transaction/action/proto.rs index 0c0ee4c4cd2..3d4d9fbfaf2 100644 --- a/rust/lance-table/src/transaction/action/proto.rs +++ b/rust/lance-table/src/transaction/action/proto.rs @@ -4,7 +4,7 @@ //! The envelope around the per-action protobuf encodings. //! //! Each action encodes itself, in its own module; this module carries the -//! [`Ref`], [`UserOperation`], and [`UserAction`] wrappers, the dispatch over +//! [`Ref`], [`CompositeOperation`], and [`UserAction`] wrappers, the dispatch over //! the `oneof`, and the helpers the per-action conversions share. //! //! Reading is fail-closed: an action this build does not implement is an error, @@ -12,7 +12,7 @@ //! transactions with `try_collect`, so a transaction carrying an unknown action //! must abort the commit rather than be treated as a no-op. -use super::{Action, Ref, UserAction, UserOperation}; +use super::{Action, CompositeOperation, Ref, UserAction}; use crate::format::pb; use lance_core::{Error, Result}; @@ -64,10 +64,9 @@ impl TryFrom for Ref { } } -impl From<&UserOperation> for pb::UserOperation { - fn from(value: &UserOperation) -> Self { +impl From<&CompositeOperation> for pb::CompositeOperation { + fn from(value: &CompositeOperation) -> Self { Self { - description: value.description.clone(), // uuid and read_version mirror the enclosing Transaction and are // stamped in by its conversion. uuid: String::new(), @@ -77,12 +76,11 @@ impl From<&UserOperation> for pb::UserOperation { } } -impl TryFrom for UserOperation { +impl TryFrom for CompositeOperation { type Error = Error; - fn try_from(message: pb::UserOperation) -> Result { + fn try_from(message: pb::CompositeOperation) -> Result { Ok(Self { - description: message.description, actions: message .actions .into_iter() @@ -288,17 +286,14 @@ mod tests { } #[test] - fn test_user_operation_round_trips() { - let operation = UserOperation::new( - "compound commit", - vec![ - UserAction::new("everything", all_actions()), - UserAction::new("nothing", vec![]), - ], - ); + fn test_composite_operation_round_trips() { + let operation = CompositeOperation::new(vec![ + UserAction::new("everything", all_actions()), + UserAction::new("nothing", vec![]), + ]); - let message = pb::UserOperation::from(&operation); - let round_tripped = UserOperation::try_from(message).unwrap(); + let message = pb::CompositeOperation::from(&operation); + let round_tripped = CompositeOperation::try_from(message).unwrap(); assert_eq!(round_tripped, operation); } diff --git a/rust/lance-table/src/transaction/action/reset_table.rs b/rust/lance-table/src/transaction/action/reset_table.rs index 516e137c5ed..8f46cff0fe0 100644 --- a/rust/lance-table/src/transaction/action/reset_table.rs +++ b/rust/lance-table/src/transaction/action/reset_table.rs @@ -134,13 +134,12 @@ mod tests { #[test] fn test_reset_conflicts_with_everything() { - use crate::transaction::action::{Footprint, UserAction, UserOperation}; + use crate::transaction::action::{CompositeOperation, Footprint, UserAction}; let footprint = |actions| { - Footprint::from(&UserOperation::new( - "test", - vec![UserAction::new("step", actions)], - )) + Footprint::from(&CompositeOperation::new(vec![UserAction::new( + "step", actions, + )])) }; let reset = footprint(vec![reset()]); diff --git a/rust/lance-table/src/transaction/action/test_support.rs b/rust/lance-table/src/transaction/action/test_support.rs index 080b13c609c..6b72355be25 100644 --- a/rust/lance-table/src/transaction/action/test_support.rs +++ b/rust/lance-table/src/transaction/action/test_support.rs @@ -3,7 +3,7 @@ //! Fixtures shared by the per-action test modules. -use super::{Action, UserAction, UserOperation}; +use super::{Action, CompositeOperation, UserAction}; use crate::format::{DataFile, Fragment, IndexMetadata, Manifest}; use crate::transaction::test_support::{default_build_config, sample_manifest}; use crate::transaction::{Operation, Transaction}; @@ -23,10 +23,9 @@ pub(super) fn apply_with_indices( ) -> Result<(Manifest, Vec)> { let transaction = Transaction::new( manifest.version, - Operation::UserOperation(UserOperation::new( - "test", - vec![UserAction::new("step", actions)], - )), + Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( + "step", actions, + )])), None, ); transaction.build_manifest(Some(manifest), indices, "tx.txn", &default_build_config()) diff --git a/rust/lance-table/src/transaction/action/translate.rs b/rust/lance-table/src/transaction/action/translate.rs index be991de5557..1e7f2d076e9 100644 --- a/rust/lance-table/src/transaction/action/translate.rs +++ b/rust/lance-table/src/transaction/action/translate.rs @@ -176,7 +176,7 @@ mod tests { }; use crate::rowids::{RowIdSequence, write_row_ids}; use crate::transaction::Transaction; - use crate::transaction::action::UserOperation; + use crate::transaction::action::CompositeOperation; use crate::transaction::test_support::{ default_build_config, make_stable_row_id_manifest, sample_manifest, }; @@ -190,7 +190,7 @@ mod tests { let actions = Vec::::try_from(&operation).unwrap(); let (translated, translated_indices) = build( manifest, - Operation::UserOperation(UserOperation::new("translated", actions)), + Operation::CompositeOperation(CompositeOperation::new(actions)), ); // Data files are addressed by field, so the two paths are allowed to @@ -388,7 +388,7 @@ mod tests { let actions = Vec::::try_from(&operation).unwrap(); let error = Transaction::new( manifest.version, - Operation::UserOperation(UserOperation::new("translated", actions)), + Operation::CompositeOperation(CompositeOperation::new(actions)), None, ) .build_manifest( diff --git a/rust/lance-table/src/transaction/conflicts.rs b/rust/lance-table/src/transaction/conflicts.rs index 57f384e24fc..e13e4950f07 100644 --- a/rust/lance-table/src/transaction/conflicts.rs +++ b/rust/lance-table/src/transaction/conflicts.rs @@ -868,8 +868,8 @@ impl PartialEq for Operation { // around. It is never equal to a legacy operation: equality here // answers "is the operation I am holding the one already // committed?", and a translated operation is a different commit. - (Self::UserOperation(a), Self::UserOperation(b)) => a == b, - (Self::UserOperation(_), _) | (_, Self::UserOperation(_)) => false, + (Self::CompositeOperation(a), Self::CompositeOperation(b)) => a == b, + (Self::CompositeOperation(_), _) | (_, Self::CompositeOperation(_)) => false, (Self::DataOverlay { .. }, _) | (_, Self::DataOverlay { .. }) => false, } } diff --git a/rust/lance-table/src/transaction/manifest_build.rs b/rust/lance-table/src/transaction/manifest_build.rs index cac9bcd17c2..951106e2f30 100644 --- a/rust/lance-table/src/transaction/manifest_build.rs +++ b/rust/lance-table/src/transaction/manifest_build.rs @@ -463,9 +463,9 @@ impl Transaction { config: &ManifestBuildConfig, read_version_state: Option>, ) -> Result<(Manifest, Vec)> { - if let Operation::UserOperation(user_operation) = &self.operation { + if let Operation::CompositeOperation(composite_operation) = &self.operation { return self.build_manifest_from_actions( - user_operation, + composite_operation, current_manifest, current_indices, transaction_file_path, @@ -1297,7 +1297,7 @@ impl Transaction { // Base paths are handled in the manifest creation section below final_fragments.extend(maybe_existing_fragments?.clone()); } - Operation::UserOperation(_) => { + Operation::CompositeOperation(_) => { // Handled by build_manifest_from_actions before this match. return Err(Error::internal( "an action-based operation reached the legacy manifest build".to_string(), diff --git a/rust/lance-table/src/transaction/operation.rs b/rust/lance-table/src/transaction/operation.rs index a8318187aad..df6356630a2 100644 --- a/rust/lance-table/src/transaction/operation.rs +++ b/rust/lance-table/src/transaction/operation.rs @@ -14,7 +14,7 @@ use crate::format::overlay::DataOverlayFile; use crate::format::{BasePath, DataFile, Fragment, IndexFile, IndexMetadata}; use crate::system_index::mem_wal::CompactedSsTable; use crate::transaction::UpdateMap; -use crate::transaction::action::UserOperation; +use crate::transaction::action::CompositeOperation; use lance_core::datatypes::Schema; use lance_core::deepsize::DeepSizeOf; use roaring::RoaringBitmap; @@ -228,7 +228,7 @@ pub enum Operation { /// Unlike the variants above, this one is not a single named change -- it is /// the composable form the others decompose into. See /// [`super::action`] for the vocabulary and its stability caveats. - UserOperation(UserOperation), + CompositeOperation(CompositeOperation), } #[derive(Debug, Clone, PartialEq, DeepSizeOf)] @@ -279,7 +279,7 @@ impl std::fmt::Display for Operation { Self::Clone { .. } => write!(f, "Clone"), Self::UpdateMemWalState { .. } => write!(f, "UpdateMemWalState"), Self::UpdateBases { .. } => write!(f, "UpdateBases"), - Self::UserOperation(_) => write!(f, "UserOperation"), + Self::CompositeOperation(_) => write!(f, "CompositeOperation"), } } } @@ -339,7 +339,7 @@ impl Operation { Self::UpdateMemWalState { .. } => "UpdateMemWalState", Self::Clone { .. } => "Clone", Self::UpdateBases { .. } => "UpdateBases", - Self::UserOperation(_) => "UserOperation", + Self::CompositeOperation(_) => "CompositeOperation", } } } diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs index 306188b1fb6..278c606db92 100644 --- a/rust/lance-table/src/transaction/proto.rs +++ b/rust/lance-table/src/transaction/proto.rs @@ -12,7 +12,7 @@ use crate::format::key_existence::KeyExistenceFilter; use crate::format::pb; use crate::format::{BasePath, Fragment, IndexFile, IndexMetadata, overlay::DataOverlayFile}; use crate::system_index::mem_wal::CompactedSsTable; -use crate::transaction::action::UserOperation; +use crate::transaction::action::CompositeOperation; use crate::transaction::{ DataOverlayGroup, DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, Transaction, UpdateMap, UpdateMapEntry, UpdateMode, UpdatedFragmentOffsets, translate_config_updates, @@ -417,14 +417,14 @@ impl TryFrom for Transaction { .map(DataOverlayGroup::try_from) .collect::>>()?, }, - Some(pb::transaction::Operation::UserOperation(user_operation)) => { + Some(pb::transaction::Operation::CompositeOperation(composite_operation)) => { // Action-based transactions (Transaction V2) are a draft wire // format (OSS-1530). Parsing is fail-closed: an action this build // does not implement is an error, never a skipped element. // load_and_sort_new_transactions collects concurrent transactions // with try_collect, so such a transaction aborts the commit rather // than being silently treated as a no-op. Do NOT make this lenient. - Operation::UserOperation(UserOperation::try_from(user_operation)?) + Operation::CompositeOperation(CompositeOperation::try_from(composite_operation)?) } None => { return Err(Error::internal( @@ -729,14 +729,14 @@ impl From<&Transaction> for pb::Transaction { .collect::>(), }) } - Operation::UserOperation(user_operation) => { - let mut message = pb::UserOperation::from(user_operation); + Operation::CompositeOperation(composite_operation) => { + let mut message = pb::CompositeOperation::from(composite_operation); // The operation's identity and read version are the enclosing // transaction's; the wire carries them in both places so a // squashed operation keeps its own provenance. message.uuid = value.uuid.clone(); message.read_version = value.read_version; - pb::transaction::Operation::UserOperation(message) + pb::transaction::Operation::CompositeOperation(message) } }; @@ -889,14 +889,13 @@ mod tests { } #[test] - fn test_user_operation_round_trips_through_transaction() { + fn test_composite_operation_round_trips_through_transaction() { let uuid = Uuid::new_v4().to_string(); let transaction = Transaction { read_version: 4, uuid: uuid.clone(), - operation: Operation::UserOperation(UserOperation::new( - "INSERT INTO t VALUES (1)", - vec![UserAction::new( + operation: Operation::CompositeOperation(CompositeOperation::new(vec![ + UserAction::new( "append batch", vec![Action::AddFragment(AddFragment { local: 0, @@ -906,8 +905,8 @@ mod tests { created_at_version_meta: None, data_change: true, })], - )], - )), + ), + ])), tag: None, transaction_properties: None, }; @@ -916,11 +915,11 @@ mod tests { // The operation repeats the envelope's identity so a squashed operation // keeps the provenance of the commit it came from. match &message.operation { - Some(pb::transaction::Operation::UserOperation(user_operation)) => { - assert_eq!(user_operation.uuid, uuid); - assert_eq!(user_operation.read_version, 4); + Some(pb::transaction::Operation::CompositeOperation(composite_operation)) => { + assert_eq!(composite_operation.uuid, uuid); + assert_eq!(composite_operation.read_version, 4); } - other => panic!("expected UserOperation, got {other:?}"), + other => panic!("expected CompositeOperation, got {other:?}"), } assert_eq!(Transaction::try_from(message).unwrap(), transaction); @@ -936,9 +935,8 @@ mod tests { let message = pb::Transaction { read_version: 1, uuid: Uuid::new_v4().to_string(), - operation: Some(pb::transaction::Operation::UserOperation( - pb::UserOperation { - description: "MERGE INTO t".to_string(), + operation: Some(pb::transaction::Operation::CompositeOperation( + pb::CompositeOperation { uuid: Uuid::new_v4().to_string(), read_version: 1, actions: vec![pb::UserAction { diff --git a/rust/lance/src/io/commit/conflict_resolver.rs b/rust/lance/src/io/commit/conflict_resolver.rs index 03b708f9094..3085cc69064 100644 --- a/rust/lance/src/io/commit/conflict_resolver.rs +++ b/rust/lance/src/io/commit/conflict_resolver.rs @@ -16,7 +16,7 @@ use lance_index::mem_wal::{CompactedSsTable, MEM_WAL_INDEX_NAME}; use lance_select::{RowAddrTreeMap, RowSetOps}; use lance_table::format::IndexMetadata; use lance_table::format::overlay::OverlayCoverage; -use lance_table::transaction::action::{Footprint, UserAction, UserOperation}; +use lance_table::transaction::action::{CompositeOperation, Footprint, UserAction}; use lance_table::{format::Fragment, io::deletion::write_deletion_file}; use roaring::RoaringBitmap; use std::{ @@ -94,7 +94,7 @@ impl<'a> TransactionRebase<'a> { // An action set can modify fragments, but conflicts against it are // settled by comparing footprints, which are derived from the // actions rather than from collected rebase state. - | Operation::UserOperation(_) => Ok(Self { + | Operation::CompositeOperation(_) => Ok(Self { transaction, affected_rows, initial_fragments: HashMap::new(), @@ -319,7 +319,9 @@ impl<'a> TransactionRebase<'a> { Operation::UpdateBases { .. } => { self.check_add_bases_txn(other_transaction, other_version) } - Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), + Operation::CompositeOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } } } @@ -356,7 +358,7 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => { + Operation::CompositeOperation(_) => { self.check_action_txn(other_transaction, other_version) } Operation::CreateIndex { .. } @@ -515,7 +517,7 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => { + Operation::CompositeOperation(_) => { self.check_action_txn(other_transaction, other_version) } Operation::CreateIndex { .. } @@ -683,7 +685,7 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => { + Operation::CompositeOperation(_) => { self.check_action_txn(other_transaction, other_version) } Operation::Append { .. } @@ -887,7 +889,7 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => { + Operation::CompositeOperation(_) => { self.check_action_txn(other_transaction, other_version) } // Rewrite is only compatible with operations that don't touch @@ -1082,7 +1084,9 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), + Operation::CompositeOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } Operation::Overwrite { .. } => { if self .transaction @@ -1134,7 +1138,9 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), + Operation::CompositeOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } // Append is not compatible with any operation that completely // overwrites the schema. Operation::Overwrite { .. } @@ -1167,7 +1173,7 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => { + Operation::CompositeOperation(_) => { self.check_action_txn(other_transaction, other_version) } Operation::Append { .. } @@ -1350,7 +1356,9 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), + Operation::CompositeOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } Operation::Append { .. } | Operation::CreateIndex { .. } | Operation::ReserveFragments { .. } @@ -1451,7 +1459,9 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), + Operation::CompositeOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } // See the MemWAL exception in check_create_index_txn. Operation::CreateIndex { new_indices, .. } => { if new_indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME) { @@ -1491,7 +1501,9 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), + Operation::CompositeOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } Operation::Append { .. } | Operation::Delete { .. } | Operation::Overwrite { .. } @@ -1521,7 +1533,9 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), + Operation::CompositeOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } Operation::Overwrite { .. } | Operation::Restore { .. } => { Err(self.incompatible_conflict_err(other_transaction, other_version)) } @@ -1550,7 +1564,9 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => self.check_action_txn(other_transaction, other_version), + Operation::CompositeOperation(_) => { + self.check_action_txn(other_transaction, other_version) + } // Project is compatible with anything that doesn't change the schema Operation::Append { .. } | Operation::Update { .. } @@ -1589,7 +1605,7 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => { + Operation::CompositeOperation(_) => { self.check_action_txn(other_transaction, other_version) } Operation::Overwrite { .. } => { @@ -1657,7 +1673,7 @@ impl<'a> TransactionRebase<'a> { match &other_transaction.operation { // A concurrent action-based transaction is compared by action // footprint rather than by operation pair. - Operation::UserOperation(_) => { + Operation::CompositeOperation(_) => { self.check_action_txn(other_transaction, other_version) } Operation::UpdateMemWalState { @@ -1818,7 +1834,7 @@ impl<'a> TransactionRebase<'a> { // minted ids are allocated when the actions are applied, against // whichever manifest they land on, and its committed references // name coordinates that do not move. - | Operation::UserOperation(_) => Ok(self.transaction), + | Operation::CompositeOperation(_) => Ok(self.transaction), } } @@ -2337,10 +2353,12 @@ fn overlay_group_coverage(group: &DataOverlayGroup) -> RoaringBitmap { /// either side needing an entry in the operation-pair matrix. fn footprint_of(operation: &Operation) -> Option { match operation { - Operation::UserOperation(user_operation) => Some(Footprint::from(user_operation)), + Operation::CompositeOperation(composite_operation) => { + Some(Footprint::from(composite_operation)) + } other => Vec::::try_from(other) .ok() - .map(|actions| Footprint::from(&UserOperation::new(other.name(), actions))), + .map(|actions| Footprint::from(&CompositeOperation::new(actions))), } } @@ -4270,10 +4288,9 @@ mod tests { fn action_txn(actions: Vec) -> Transaction { Transaction::new_from_version( 1, - Operation::UserOperation(UserOperation::new( - "test", - vec![UserAction::new("step", actions)], - )), + Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( + "step", actions, + )])), ) } @@ -4669,7 +4686,7 @@ mod tests { | Operation::UpdateBases { .. } | Operation::Restore { .. } | Operation::UpdateMemWalState { .. } - | Operation::UserOperation(_) => Box::new(std::iter::empty()), + | Operation::CompositeOperation(_) => Box::new(std::iter::empty()), Operation::Delete { updated_fragments, deleted_fragment_ids, diff --git a/rust/lance/tests/composite_transaction.rs b/rust/lance/tests/composite_transaction.rs index 7d967cd012f..d93be6e38ff 100644 --- a/rust/lance/tests/composite_transaction.rs +++ b/rust/lance/tests/composite_transaction.rs @@ -22,8 +22,8 @@ use lance::Dataset; use lance::dataset::{CommitBuilder, InsertBuilder, WriteParams}; use lance_table::format::DataFile; use lance_table::transaction::action::{ - Action, AddDataFile, AddField, AddFragment, DropField, Ref, TombstoneFieldData, UserAction, - UserOperation, + Action, AddDataFile, AddField, AddFragment, CompositeOperation, DropField, Ref, + TombstoneFieldData, UserAction, }; use lance_table::transaction::{Operation, Transaction}; @@ -54,10 +54,9 @@ async fn commit(dataset: Dataset, actions: Vec) -> Dataset { CommitBuilder::new(Arc::new(dataset)) .execute(Transaction::new( read_version, - Operation::UserOperation(UserOperation::new( - "composite", - vec![UserAction::new("step", actions)], - )), + Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( + "step", actions, + )])), None, )) .await @@ -251,10 +250,10 @@ async fn test_two_action_sets_on_disjoint_coordinates_both_commit() { let first = CommitBuilder::new(dataset.clone()) .execute(Transaction::new( read_version, - Operation::UserOperation(UserOperation::new( - "first", - vec![UserAction::new("step", append(0))], - )), + Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( + "step", + append(0), + )])), None, )) .await @@ -266,10 +265,10 @@ async fn test_two_action_sets_on_disjoint_coordinates_both_commit() { let second = CommitBuilder::new(dataset) .execute(Transaction::new( read_version, - Operation::UserOperation(UserOperation::new( - "second", - vec![UserAction::new("step", append(0))], - )), + Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( + "step", + append(0), + )])), None, )) .await @@ -288,17 +287,14 @@ async fn test_two_action_sets_writing_the_same_field_data_conflict() { let tombstone = || { Transaction::new( read_version, - Operation::UserOperation(UserOperation::new( - "tombstone", - vec![UserAction::new( - "step", - vec![Action::TombstoneFieldData(TombstoneFieldData { - fragment: Ref::Committed(fragment_id), - field_ids: vec![0], - data_change: true, - })], - )], - )), + Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( + "step", + vec![Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Committed(fragment_id), + field_ids: vec![0], + data_change: true, + })], + )])), None, ) }; From aa751d7eae71361273ef3ace3ae7e61845b42548 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 18 Aug 2026 12:58:09 -0700 Subject: [PATCH 18/24] docs(transaction): document when action version and data_change fields differ `AddFragment`'s version metadata fields said only "stamp at apply" without saying who would ever set them. Name the two producers that must: an update carries each row's `created_at` forward from its source fragment, and a compaction rechunks both sequences off the fragments it merged. `TombstoneFieldData::data_change` deferred to `AddFragment`'s doc, which does not cover it. State the case directly: a tombstone paired with a re-add in the same operation moves bytes without changing values. Strikes an over-long comment in `conflicts.rs`. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance-table/src/transaction/action/add_fragment.rs | 9 ++++++++- .../src/transaction/action/tombstone_field_data.rs | 6 +++++- rust/lance-table/src/transaction/conflicts.rs | 5 ----- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/rust/lance-table/src/transaction/action/add_fragment.rs b/rust/lance-table/src/transaction/action/add_fragment.rs index e172eaa57de..75ebb099cad 100644 --- a/rust/lance-table/src/transaction/action/add_fragment.rs +++ b/rust/lance-table/src/transaction/action/add_fragment.rs @@ -26,7 +26,14 @@ pub struct AddFragment { /// on datasets that have them but where the ids are assigned at apply. pub row_id_meta: Option, /// Per-row version metadata, carried exactly as on - /// [`Fragment`](crate::format::Fragment). `None` means "stamp at apply". + /// [`Fragment`](crate::format::Fragment). + /// + /// `None` means "stamp at apply", which fills both with a uniform sequence + /// at the commit version. That is right for an append, but not for the two + /// producers whose rows carry versions from before this commit, so they set + /// the fields explicitly: an update resolves each row's `created_at` from + /// the fragment the row came from, and a compaction rechunks both sequences + /// off the fragments it merged, since moving a row does not update it. pub last_updated_at_version_meta: Option, pub created_at_version_meta: Option, /// `false` marks a pure rearrangement, e.g. a compaction rewrite. diff --git a/rust/lance-table/src/transaction/action/tombstone_field_data.rs b/rust/lance-table/src/transaction/action/tombstone_field_data.rs index 59b0e2f8ad1..0d6a57b0bcd 100644 --- a/rust/lance-table/src/transaction/action/tombstone_field_data.rs +++ b/rust/lance-table/src/transaction/action/tombstone_field_data.rs @@ -22,7 +22,11 @@ pub struct TombstoneFieldData { pub fragment: Ref, /// Committed field ids whose current backing is tombstoned. pub field_ids: Vec, - /// See [`AddFragment::data_change`](super::AddFragment::data_change). + /// `false` marks a tombstone whose data is re-added by an + /// [`AddDataFile`](super::AddDataFile) for the same fields in the same + /// operation, i.e. a re-encode that moves the bytes without changing any + /// row's value. A tombstone with no matching re-add nulls the column out + /// and is a data change. pub data_change: bool, } diff --git a/rust/lance-table/src/transaction/conflicts.rs b/rust/lance-table/src/transaction/conflicts.rs index e13e4950f07..dc929bd9adf 100644 --- a/rust/lance-table/src/transaction/conflicts.rs +++ b/rust/lance-table/src/transaction/conflicts.rs @@ -863,11 +863,6 @@ impl PartialEq for Operation { std::mem::discriminant(self) == std::mem::discriminant(other) } (Self::DataOverlay { groups: a }, Self::DataOverlay { groups: b }) => compare_vec(a, b), - // A V2 operation is an ordered list, so unlike the operations above - // it compares element-wise with no order-insensitivity to work - // around. It is never equal to a legacy operation: equality here - // answers "is the operation I am holding the one already - // committed?", and a translated operation is a different commit. (Self::CompositeOperation(a), Self::CompositeOperation(b)) => a == b, (Self::CompositeOperation(_), _) | (_, Self::CompositeOperation(_)) => false, (Self::DataOverlay { .. }, _) | (_, Self::DataOverlay { .. }) => false, From f079ca2f7a4011301200dd94658eb411e4b217ea Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 18 Aug 2026 13:02:01 -0700 Subject: [PATCH 19/24] refactor(transaction): generate the action dispatch from one list Adding an action meant editing five per-variant matches -- the enum, `name`, `is_data_change`, `apply`, `footprint` -- plus both directions of the wire encoding, in two files. Nothing but review caught a variant handled inconsistently across them. `for_each_action!` now holds the vocabulary as a single list of variant names, and the enum, its four forwarding methods, and both proto conversions expand from it. Adding an action is a module plus one line. The variant name is also the protobuf oneof variant and the name in errors, so those cannot drift. `is_data_change` moves into the action modules alongside `apply` and `footprint`, so an action's module answers every question about it. The grouped comments that used to sit in the central match move with it, one to each action. This is the `enum_dispatch` pattern done locally: it needs no new dependency, and it also covers the decode direction, which dispatches on the protobuf oneof tag rather than on `self` and so is out of `enum_dispatch`'s reach. Co-Authored-By: Claude Opus 5 (1M context) --- rust/lance-table/src/transaction/action.rs | 172 ++++++++---------- .../src/transaction/action/add_base.rs | 5 + .../src/transaction/action/add_data_file.rs | 4 + .../src/transaction/action/add_field.rs | 6 + .../src/transaction/action/add_fragment.rs | 4 + .../src/transaction/action/alter_field.rs | 6 + .../src/transaction/action/config_update.rs | 5 + .../src/transaction/action/drop_field.rs | 5 + .../src/transaction/action/proto.rs | 98 ++++------ .../src/transaction/action/remove_fragment.rs | 4 + .../action/reserve_fragment_ids.rs | 5 + .../src/transaction/action/reset_table.rs | 5 + .../transaction/action/set_deletion_file.rs | 4 + .../action/tombstone_field_data.rs | 4 + 14 files changed, 165 insertions(+), 162 deletions(-) diff --git a/rust/lance-table/src/transaction/action.rs b/rust/lance-table/src/transaction/action.rs index a2f26bef7b9..703adb77db0 100644 --- a/rust/lance-table/src/transaction/action.rs +++ b/rust/lance-table/src/transaction/action.rs @@ -33,6 +33,39 @@ //! contract, and a transaction carrying a [`CompositeOperation`] is rejected outright //! by libraries that predate it. +/// The action vocabulary, as one list. +/// +/// Every per-variant `match` over an [`Action`] -- the enum itself, its +/// forwarding methods, and both directions of its wire encoding -- is generated +/// from this, so an action is added by writing its module and adding one name +/// here. A module that does not supply the full set of methods fails to +/// compile. +/// +/// The variant name doubles as the protobuf `oneof` variant name and as the +/// action's name in errors and logs, so the three cannot drift apart. +/// +/// This is defined ahead of the modules below so it is in scope for all of +/// them; `macro_rules!` visibility runs from the definition to the end of the +/// enclosing module, children included. +macro_rules! for_each_action { + ($emit:ident) => { + $emit! { + AddFragment, + AddDataFile, + AddField, + AddBase, + TombstoneFieldData, + RemoveFragment, + SetDeletionFile, + AlterField, + DropField, + ReserveFragmentIds, + ResetTable, + ConfigUpdate, + } + }; +} + mod add_base; mod add_data_file; mod add_field; @@ -153,108 +186,57 @@ impl UserAction { } } -/// A single granular change to the manifest. -/// -/// The drafted vocabulary is larger than this; the variants here are the ones -/// this build implements end to end. Each one is defined, applied, and encoded -/// in the module named after it. -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub enum Action { - AddFragment(AddFragment), - AddDataFile(AddDataFile), - AddField(AddField), - AddBase(AddBase), - TombstoneFieldData(TombstoneFieldData), - RemoveFragment(RemoveFragment), - SetDeletionFile(SetDeletionFile), - AlterField(AlterField), - DropField(DropField), - ReserveFragmentIds(ReserveFragmentIds), - ResetTable(ResetTable), - ConfigUpdate(ConfigUpdate), -} - -impl Action { - pub fn name(&self) -> &'static str { - match self { - Self::AddFragment(_) => "AddFragment", - Self::AddDataFile(_) => "AddDataFile", - Self::AddField(_) => "AddField", - Self::AddBase(_) => "AddBase", - Self::TombstoneFieldData(_) => "TombstoneFieldData", - Self::RemoveFragment(_) => "RemoveFragment", - Self::SetDeletionFile(_) => "SetDeletionFile", - Self::AlterField(_) => "AlterField", - Self::DropField(_) => "DropField", - Self::ReserveFragmentIds(_) => "ReserveFragmentIds", - Self::ResetTable(_) => "ResetTable", - Self::ConfigUpdate(_) => "ConfigUpdate", +macro_rules! define_action { + ($($variant:ident,)*) => { + /// A single granular change to the manifest. + /// + /// The drafted vocabulary is larger than this; the variants here are the + /// ones this build implements end to end. Each one is defined, applied, + /// and encoded in the module named after it, and appears here only + /// because it is listed in `for_each_action!`. + #[derive(Debug, Clone, PartialEq, DeepSizeOf)] + pub enum Action { + $($variant($variant),)* } - } - /// Whether this action changes the data a reader would see, as opposed to - /// rearranging how it is stored (compaction, a segment rebuild). - /// - /// CDC and streaming consumers use this to skip commits that cannot have - /// changed any row's value. - pub fn is_data_change(&self) -> bool { - match self { - Self::AddFragment(action) => action.data_change, - Self::AddDataFile(action) => action.data_change, - Self::TombstoneFieldData(action) => action.data_change, - Self::RemoveFragment(action) => action.data_change, - Self::SetDeletionFile(action) => action.data_change, - // Dropping a field discards the values it held. - Self::DropField(_) => true, - // Other schema and base-path changes touch no row values. - // Emptying the table discards every row it held. - Self::ResetTable(_) => true, - // Reserving ids writes no rows either. - Self::AddField(_) - | Self::AddBase(_) - | Self::AlterField(_) - | Self::ReserveFragmentIds(_) - | Self::ConfigUpdate(_) => false, - } - } + impl Action { + pub fn name(&self) -> &'static str { + match self { + $(Self::$variant(_) => stringify!($variant),)* + } + } - /// Fold this action into the state the next manifest is built from. - fn apply(&self, state: &mut ApplyState) -> Result<()> { - match self { - Self::AddFragment(action) => action.apply(state), - Self::AddDataFile(action) => action.apply(state), - Self::AddField(action) => action.apply(state), - Self::AddBase(action) => action.apply(state), - Self::TombstoneFieldData(action) => action.apply(state), - Self::RemoveFragment(action) => action.apply(state), - Self::SetDeletionFile(action) => action.apply(state), - Self::AlterField(action) => action.apply(state), - Self::DropField(action) => action.apply(state), - Self::ReserveFragmentIds(action) => action.apply(state), - Self::ResetTable(action) => action.apply(state), - Self::ConfigUpdate(action) => action.apply(state), - } - } + /// Whether this action changes the data a reader would see, as + /// opposed to rearranging how it is stored (compaction, a segment + /// rebuild). + /// + /// CDC and streaming consumers use this to skip commits that cannot + /// have changed any row's value. + pub fn is_data_change(&self) -> bool { + match self { + $(Self::$variant(action) => action.is_data_change(),)* + } + } - /// Record the coordinates this action writes. - fn footprint(&self, footprint: &mut Footprint) { - match self { - Self::AddFragment(action) => action.footprint(footprint), - Self::AddDataFile(action) => action.footprint(footprint), - Self::AddField(action) => action.footprint(footprint), - Self::AddBase(action) => action.footprint(footprint), - Self::TombstoneFieldData(action) => action.footprint(footprint), - Self::RemoveFragment(action) => action.footprint(footprint), - Self::SetDeletionFile(action) => action.footprint(footprint), - Self::AlterField(action) => action.footprint(footprint), - Self::DropField(action) => action.footprint(footprint), - Self::ReserveFragmentIds(action) => action.footprint(footprint), - Self::ResetTable(action) => action.footprint(footprint), - Self::ConfigUpdate(action) => action.footprint(footprint), + /// Fold this action into the state the next manifest is built from. + fn apply(&self, state: &mut ApplyState) -> Result<()> { + match self { + $(Self::$variant(action) => action.apply(state),)* + } + } + + /// Record the coordinates this action writes. + fn footprint(&self, footprint: &mut Footprint) { + match self { + $(Self::$variant(action) => action.footprint(footprint),)* + } + } } - } + }; } +for_each_action!(define_action); + impl std::fmt::Display for Action { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.name()) diff --git a/rust/lance-table/src/transaction/action/add_base.rs b/rust/lance-table/src/transaction/action/add_base.rs index d82f163b965..98df7506d7a 100644 --- a/rust/lance-table/src/transaction/action/add_base.rs +++ b/rust/lance-table/src/transaction/action/add_base.rs @@ -40,6 +40,11 @@ impl AddBase { Ok(()) } + /// A base path is a location, not data. + pub(super) fn is_data_change(&self) -> bool { + false + } + /// The base id is minted, but the name and location are not: the manifest /// requires both to be unique, so two operations claiming either collide. pub(super) fn footprint(&self, footprint: &mut Footprint) { diff --git a/rust/lance-table/src/transaction/action/add_data_file.rs b/rust/lance-table/src/transaction/action/add_data_file.rs index 6cc141f5c7f..59c673db0b2 100644 --- a/rust/lance-table/src/transaction/action/add_data_file.rs +++ b/rust/lance-table/src/transaction/action/add_data_file.rs @@ -52,6 +52,10 @@ impl AddDataFile { Ok(()) } + pub(super) fn is_data_change(&self) -> bool { + self.data_change + } + /// The data of every committed field the file backs, in the fragment it is /// attached to. A file backing only minted fields writes nothing. pub(super) fn footprint(&self, footprint: &mut Footprint) { diff --git a/rust/lance-table/src/transaction/action/add_field.rs b/rust/lance-table/src/transaction/action/add_field.rs index a82d210cf21..4847edbe6d1 100644 --- a/rust/lance-table/src/transaction/action/add_field.rs +++ b/rust/lance-table/src/transaction/action/add_field.rs @@ -62,6 +62,12 @@ impl AddField { Ok(()) } + /// A new field starts empty; an + /// [`AddDataFile`](super::AddDataFile) is what gives it values. + pub(super) fn is_data_change(&self) -> bool { + false + } + /// Nothing: the field does not exist in the read version. Attaching it /// under a committed parent does not rewrite the parent's definition. pub(super) fn footprint(&self, _footprint: &mut Footprint) {} diff --git a/rust/lance-table/src/transaction/action/add_fragment.rs b/rust/lance-table/src/transaction/action/add_fragment.rs index 75ebb099cad..0eb3140f3c6 100644 --- a/rust/lance-table/src/transaction/action/add_fragment.rs +++ b/rust/lance-table/src/transaction/action/add_fragment.rs @@ -56,6 +56,10 @@ impl AddFragment { Ok(()) } + pub(super) fn is_data_change(&self) -> bool { + self.data_change + } + /// Nothing: the fragment does not exist in the read version, so no /// concurrent writer can be naming it. pub(super) fn footprint(&self, _footprint: &mut Footprint) {} diff --git a/rust/lance-table/src/transaction/action/alter_field.rs b/rust/lance-table/src/transaction/action/alter_field.rs index 545d07e7d87..e3380056b87 100644 --- a/rust/lance-table/src/transaction/action/alter_field.rs +++ b/rust/lance-table/src/transaction/action/alter_field.rs @@ -53,6 +53,12 @@ impl AlterField { Ok(()) } + /// Renaming or relaxing a field leaves the values alone, and the + /// rewrite a cast needs is separate actions that answer for themselves. + pub(super) fn is_data_change(&self) -> bool { + false + } + /// The field's definition. The data rewrite a cast needs is separate /// actions, which record their own coordinates. pub(super) fn footprint(&self, footprint: &mut Footprint) { diff --git a/rust/lance-table/src/transaction/action/config_update.rs b/rust/lance-table/src/transaction/action/config_update.rs index 544f8c41b16..ac1200dbb61 100644 --- a/rust/lance-table/src/transaction/action/config_update.rs +++ b/rust/lance-table/src/transaction/action/config_update.rs @@ -101,6 +101,11 @@ impl ConfigUpdate { ) } + /// The config and metadata maps hold no row values. + pub(super) fn is_data_change(&self) -> bool { + false + } + /// The keys this update names, or the whole map when it replaces one. A /// field's metadata belongs to the field, so dropping the field also /// collides with an update to it. diff --git a/rust/lance-table/src/transaction/action/drop_field.rs b/rust/lance-table/src/transaction/action/drop_field.rs index 4b03d82479f..b4b5ceba8c2 100644 --- a/rust/lance-table/src/transaction/action/drop_field.rs +++ b/rust/lance-table/src/transaction/action/drop_field.rs @@ -71,6 +71,11 @@ impl DropField { Ok(()) } + /// Dropping a field discards every value it held. + pub(super) fn is_data_change(&self) -> bool { + true + } + /// The field's definition and all of its data, which cannot be enumerated, /// so the removal is recorded as such and matched by field id. pub(super) fn footprint(&self, footprint: &mut Footprint) { diff --git a/rust/lance-table/src/transaction/action/proto.rs b/rust/lance-table/src/transaction/action/proto.rs index 3d4d9fbfaf2..6cb50b8a3e3 100644 --- a/rust/lance-table/src/transaction/action/proto.rs +++ b/rust/lance-table/src/transaction/action/proto.rs @@ -114,81 +114,45 @@ impl TryFrom for UserAction { } } -impl From<&Action> for pb::Action { - fn from(value: &Action) -> Self { - let action = match value { - Action::AddFragment(action) => pb::action::Action::AddFragment(action.into()), - Action::AddDataFile(action) => pb::action::Action::AddDataFile(action.into()), - Action::AddField(action) => pb::action::Action::AddField(action.into()), - Action::AddBase(action) => pb::action::Action::AddBase(action.into()), - Action::TombstoneFieldData(action) => { - pb::action::Action::TombstoneFieldData(action.into()) +macro_rules! define_action_proto { + ($($variant:ident,)*) => { + impl From<&Action> for pb::Action { + fn from(value: &Action) -> Self { + let action = match value { + $(Action::$variant(action) => pb::action::Action::$variant(action.into()),)* + }; + Self { + action: Some(action), + } } - Action::RemoveFragment(action) => pb::action::Action::RemoveFragment(action.into()), - Action::SetDeletionFile(action) => pb::action::Action::SetDeletionFile(action.into()), - Action::AlterField(action) => pb::action::Action::AlterField(action.into()), - Action::DropField(action) => pb::action::Action::DropField(action.into()), - Action::ReserveFragmentIds(action) => { - pb::action::Action::ReserveFragmentIds(action.into()) - } - Action::ResetTable(action) => pb::action::Action::ResetTable(action.into()), - Action::ConfigUpdate(action) => pb::action::Action::ConfigUpdate(action.into()), - }; - Self { - action: Some(action), } - } -} -impl TryFrom for Action { - type Error = Error; + impl TryFrom for Action { + type Error = Error; - fn try_from(message: pb::Action) -> Result { - match message.action { - Some(pb::action::Action::AddFragment(action)) => { - Ok(Self::AddFragment(action.try_into()?)) - } - Some(pb::action::Action::AddDataFile(action)) => { - Ok(Self::AddDataFile(action.try_into()?)) - } - Some(pb::action::Action::AddField(action)) => Ok(Self::AddField(action.try_into()?)), - Some(pb::action::Action::AddBase(action)) => Ok(Self::AddBase(action.try_into()?)), - Some(pb::action::Action::TombstoneFieldData(action)) => { - Ok(Self::TombstoneFieldData(action.try_into()?)) - } - Some(pb::action::Action::RemoveFragment(action)) => { - Ok(Self::RemoveFragment(action.try_into()?)) - } - Some(pb::action::Action::SetDeletionFile(action)) => { - Ok(Self::SetDeletionFile(action.try_into()?)) + fn try_from(message: pb::Action) -> Result { + match message.action { + $(Some(pb::action::Action::$variant(action)) => { + Ok(Self::$variant(action.try_into()?)) + })* + // The drafted vocabulary is larger than what is implemented. + // Reject rather than skip: silently dropping an action would + // apply a partial transaction. + Some(other) => Err(Error::not_supported(format!( + "the action-based transaction uses action {other:?}, which is drafted \ + but not implemented by this version of Lance", + ))), + None => Err(Error::invalid_input( + "an Action in a user operation was empty", + )), + } } - Some(pb::action::Action::AlterField(action)) => { - Ok(Self::AlterField(action.try_into()?)) - } - Some(pb::action::Action::DropField(action)) => Ok(Self::DropField(action.try_into()?)), - Some(pb::action::Action::ReserveFragmentIds(action)) => { - Ok(Self::ReserveFragmentIds(action.try_into()?)) - } - Some(pb::action::Action::ResetTable(action)) => { - Ok(Self::ResetTable(action.try_into()?)) - } - Some(pb::action::Action::ConfigUpdate(action)) => { - Ok(Self::ConfigUpdate(action.try_into()?)) - } - // The drafted vocabulary is larger than what is implemented. Reject - // rather than skip: silently dropping an action would apply a - // partial transaction. - Some(other) => Err(Error::not_supported(format!( - "the action-based transaction uses action {other:?}, which is drafted but not \ - implemented by this version of Lance", - ))), - None => Err(Error::invalid_input( - "an Action in a user operation was empty", - )), } - } + }; } +for_each_action!(define_action_proto); + #[cfg(test)] mod tests { use super::*; diff --git a/rust/lance-table/src/transaction/action/remove_fragment.rs b/rust/lance-table/src/transaction/action/remove_fragment.rs index 65955398bed..43cd41fdd89 100644 --- a/rust/lance-table/src/transaction/action/remove_fragment.rs +++ b/rust/lance-table/src/transaction/action/remove_fragment.rs @@ -30,6 +30,10 @@ impl RemoveFragment { Ok(()) } + pub(super) fn is_data_change(&self) -> bool { + self.data_change + } + /// Every coordinate inside the fragment, which cannot be enumerated, so the /// removal is recorded as such and matched by fragment id. pub(super) fn footprint(&self, footprint: &mut Footprint) { diff --git a/rust/lance-table/src/transaction/action/reserve_fragment_ids.rs b/rust/lance-table/src/transaction/action/reserve_fragment_ids.rs index d5c71e4704c..3e4697eb6e3 100644 --- a/rust/lance-table/src/transaction/action/reserve_fragment_ids.rs +++ b/rust/lance-table/src/transaction/action/reserve_fragment_ids.rs @@ -28,6 +28,11 @@ impl ReserveFragmentIds { Ok(()) } + /// A reserved id backs no rows until something is written to it. + pub(super) fn is_data_change(&self) -> bool { + false + } + /// Nothing. Ids come off a monotonic counter, so two operations reserving /// at once get disjoint ranges rather than colliding. pub(super) fn footprint(&self, _footprint: &mut Footprint) {} diff --git a/rust/lance-table/src/transaction/action/reset_table.rs b/rust/lance-table/src/transaction/action/reset_table.rs index 8f46cff0fe0..54504f94f6b 100644 --- a/rust/lance-table/src/transaction/action/reset_table.rs +++ b/rust/lance-table/src/transaction/action/reset_table.rs @@ -30,6 +30,11 @@ impl ResetTable { Ok(()) } + /// Emptying the table discards every row it held. + pub(super) fn is_data_change(&self) -> bool { + true + } + /// Everything. A reset writes every coordinate there is, including ones a /// concurrent set would only mint, so it takes the table exclusively rather /// than enumerating them. diff --git a/rust/lance-table/src/transaction/action/set_deletion_file.rs b/rust/lance-table/src/transaction/action/set_deletion_file.rs index 4406681c83e..7434bcf819a 100644 --- a/rust/lance-table/src/transaction/action/set_deletion_file.rs +++ b/rust/lance-table/src/transaction/action/set_deletion_file.rs @@ -36,6 +36,10 @@ impl SetDeletionFile { Ok(()) } + pub(super) fn is_data_change(&self) -> bool { + self.data_change + } + /// The fragment's deletions, which is a distinct coordinate from the data of /// any field in it: deleting rows and re-encoding a column commute. pub(super) fn footprint(&self, footprint: &mut Footprint) { diff --git a/rust/lance-table/src/transaction/action/tombstone_field_data.rs b/rust/lance-table/src/transaction/action/tombstone_field_data.rs index 0d6a57b0bcd..505daab5433 100644 --- a/rust/lance-table/src/transaction/action/tombstone_field_data.rs +++ b/rust/lance-table/src/transaction/action/tombstone_field_data.rs @@ -67,6 +67,10 @@ impl TombstoneFieldData { Ok(()) } + pub(super) fn is_data_change(&self) -> bool { + self.data_change + } + /// The data of each named field in this fragment, and nothing else: another /// field's data in the same fragment is untouched. pub(super) fn footprint(&self, footprint: &mut Footprint) { From badd4f51f50b449dce9364efb732bbc4a6f0d4fa Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 18 Aug 2026 13:06:01 -0700 Subject: [PATCH 20/24] test(lance): move composite transaction tests into the library A dedicated integration test gets its own binary, which links every dependency of the crate. Fold the six tests into the existing `dataset::tests::dataset_transactions` module as a `composite` submodule and delete the standalone target. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/dataset/tests/dataset_transactions.rs | 325 ++++++++++++++++++ rust/lance/tests/composite_transaction.rs | 316 ----------------- 2 files changed, 325 insertions(+), 316 deletions(-) delete mode 100644 rust/lance/tests/composite_transaction.rs diff --git a/rust/lance/src/dataset/tests/dataset_transactions.rs b/rust/lance/src/dataset/tests/dataset_transactions.rs index 5b9c76bc90c..77f0f3ae72a 100644 --- a/rust/lance/src/dataset/tests/dataset_transactions.rs +++ b/rust/lance/src/dataset/tests/dataset_transactions.rs @@ -1395,3 +1395,328 @@ async fn test_alter_columns_materializes_fresh_field_id_in_every_fragment() { let batch = dataset.scan().try_into_batch().await.unwrap(); assert_eq!(batch["a"].as_primitive::().values(), &[1, 2]); } + +mod composite { + //! End-to-end coverage for committing an action set against a real dataset. + //! + //! Each of these is a single commit that does work a named operation would + //! have needed several commits for: a fragment is added and then modified, a + //! field is added and then filled, all inside one version. That is what the + //! action vocabulary buys -- steps that reference each other's minted ids can + //! be squashed into one atomic manifest change. + //! + //! The commit path checks that a referenced data file exists, so these tests + //! point their actions at files the fixture dataset already wrote. The + //! resulting datasets are inspected through their manifests rather than read -- + //! the files hold the wrong columns for where they end up attached. + + use std::sync::Arc; + + use crate::Dataset; + use crate::dataset::{CommitBuilder, InsertBuilder, WriteParams}; + use arrow_array::{Int32Array, RecordBatch}; + use arrow_schema::{DataType, Field, Schema}; + use lance_table::format::DataFile; + use lance_table::transaction::action::{ + Action, AddDataFile, AddField, AddFragment, CompositeOperation, DropField, Ref, + TombstoneFieldData, UserAction, + }; + use lance_table::transaction::{Operation, Transaction}; + + /// A two-fragment dataset, so its two data files can stand in for the files an + /// action set would otherwise have had to write. + async fn test_dataset(enable_stable_row_ids: bool) -> Dataset { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let data = + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from_iter_values(0..10))]) + .unwrap(); + + InsertBuilder::new("memory://") + .with_params(&WriteParams { + enable_stable_row_ids, + max_rows_per_file: 5, + ..Default::default() + }) + .execute(vec![data]) + .await + .unwrap() + } + + fn existing_data_file(dataset: &Dataset, fragment: usize) -> DataFile { + dataset.fragments()[fragment].files[0].clone() + } + + async fn commit(dataset: Dataset, actions: Vec) -> Dataset { + let read_version = dataset.version().version; + CommitBuilder::new(Arc::new(dataset)) + .execute(Transaction::new( + read_version, + Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( + "step", actions, + )])), + None, + )) + .await + .unwrap() + } + + #[tokio::test] + async fn test_one_commit_adds_a_fragment_and_then_modifies_it() { + let dataset = test_dataset(false).await; + let before = dataset.version().version; + let first_file = existing_data_file(&dataset, 0); + let second_file = existing_data_file(&dataset, 1); + let second_path = second_file.path.clone(); + + let dataset = commit( + dataset, + vec![ + Action::AddFragment(AddFragment { + local: 0, + physical_rows: 4, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Local(0), + file: first_file, + field_ids: vec![Ref::Committed(0)], + data_change: true, + }), + // The same commit then replaces the data it just added, naming the + // fragment by the token it was minted under. + Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Local(0), + field_ids: vec![0], + data_change: true, + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Local(0), + file: second_file, + field_ids: vec![Ref::Committed(0)], + data_change: true, + }), + ], + ) + .await; + + assert_eq!(dataset.version().version, before + 1); + let fragments = dataset.fragments(); + assert_eq!(fragments.len(), 3); + let added = fragments.last().unwrap(); + assert_eq!(added.physical_rows, Some(4)); + // The tombstoned file is gone; only the replacement survives the commit. + let paths = added + .files + .iter() + .map(|file| file.path.clone()) + .collect::>(); + assert_eq!(paths, vec![second_path]); + } + + #[tokio::test] + async fn test_one_commit_adds_a_field_and_then_fills_it() { + let dataset = test_dataset(false).await; + let fragment_id = dataset.fragments()[0].id; + let file = existing_data_file(&dataset, 1); + let path = file.path.clone(); + + let dataset = commit( + dataset, + vec![ + Action::AddField(AddField { + local: 0, + parent: None, + def: lance_core::datatypes::Field::try_from(Field::new( + "b", + DataType::Int32, + true, + )) + .unwrap(), + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Committed(fragment_id), + file, + field_ids: vec![Ref::Local(0)], + data_change: true, + }), + ], + ) + .await; + + let field = dataset.schema().field("b").expect("field b was added"); + let fragment = &dataset.fragments()[0]; + let added = fragment + .files + .iter() + .find(|file| file.path == path) + .expect("the new field's data file was attached"); + // The file points at the id the commit minted, which the caller never knew. + assert_eq!(added.fields.as_ref(), &[field.id]); + } + + #[tokio::test] + async fn test_one_commit_swaps_a_field_for_a_new_one() { + let dataset = test_dataset(false).await; + let fragment_id = dataset.fragments()[0].id; + let file = existing_data_file(&dataset, 1); + + let dataset = commit( + dataset, + vec![ + Action::DropField(DropField { field: 0 }), + Action::AddField(AddField { + local: 0, + parent: None, + def: lance_core::datatypes::Field::try_from(Field::new( + "a", + DataType::Int64, + true, + )) + .unwrap(), + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Committed(fragment_id), + file, + field_ids: vec![Ref::Local(0)], + data_change: true, + }), + ], + ) + .await; + + // Dropping "a" and adding a new "a" of a different type is one version, and + // the new field gets a fresh id rather than inheriting the dropped one. + let schema = dataset.schema(); + assert_eq!(schema.fields.len(), 1); + let field = schema.field("a").unwrap(); + assert_ne!(field.id, 0); + assert_eq!(field.logical_type.to_string(), "int64"); + } + + #[tokio::test] + async fn test_one_commit_assigns_row_ids_to_the_fragments_it_mints() { + let dataset = test_dataset(true).await; + let next_row_id = dataset.manifest().next_row_id; + assert_eq!(next_row_id, 10); + + let dataset = commit( + dataset, + vec![ + Action::AddFragment(AddFragment { + local: 0, + physical_rows: 4, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }), + Action::AddFragment(AddFragment { + local: 1, + physical_rows: 6, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + }), + ], + ) + .await; + + assert_eq!(dataset.manifest().next_row_id, 20); + for fragment in dataset.fragments().iter().skip(2) { + assert!( + fragment.row_id_meta.is_some(), + "fragment {} was minted without row ids", + fragment.id + ); + assert!(fragment.created_at_version_meta.is_some()); + } + } + + #[tokio::test] + async fn test_two_action_sets_on_disjoint_coordinates_both_commit() { + let dataset = Arc::new(test_dataset(false).await); + let read_version = dataset.version().version; + + let append = |local| { + vec![Action::AddFragment(AddFragment { + local, + physical_rows: 4, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + data_change: true, + })] + }; + + let first = CommitBuilder::new(dataset.clone()) + .execute(Transaction::new( + read_version, + Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( + "step", + append(0), + )])), + None, + )) + .await + .unwrap(); + + // The second commit still reads the original version, so it has to be + // checked against the first. Both only mint, so neither writes anything + // the other does. + let second = CommitBuilder::new(dataset) + .execute(Transaction::new( + read_version, + Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( + "step", + append(0), + )])), + None, + )) + .await + .unwrap(); + + assert_eq!(second.version().version, first.version().version + 1); + assert_eq!(second.fragments().len(), 4); + } + + #[tokio::test] + async fn test_two_action_sets_writing_the_same_field_data_conflict() { + let dataset = Arc::new(test_dataset(false).await); + let read_version = dataset.version().version; + let fragment_id = dataset.fragments()[0].id; + + let tombstone = || { + Transaction::new( + read_version, + Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( + "step", + vec![Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Committed(fragment_id), + field_ids: vec![0], + data_change: true, + })], + )])), + None, + ) + }; + + CommitBuilder::new(dataset.clone()) + .execute(tombstone()) + .await + .unwrap(); + + let error = CommitBuilder::new(dataset) + .with_max_retries(0) + .execute(tombstone()) + .await + .unwrap_err(); + assert!( + error.to_string().contains("preempted"), + "unexpected error: {error}" + ); + } +} diff --git a/rust/lance/tests/composite_transaction.rs b/rust/lance/tests/composite_transaction.rs deleted file mode 100644 index d93be6e38ff..00000000000 --- a/rust/lance/tests/composite_transaction.rs +++ /dev/null @@ -1,316 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! End-to-end coverage for committing an action set against a real dataset. -//! -//! Each of these is a single commit that does work a named operation would -//! have needed several commits for: a fragment is added and then modified, a -//! field is added and then filled, all inside one version. That is what the -//! action vocabulary buys -- steps that reference each other's minted ids can -//! be squashed into one atomic manifest change. -//! -//! The commit path checks that a referenced data file exists, so these tests -//! point their actions at files the fixture dataset already wrote. The -//! resulting datasets are inspected through their manifests rather than read -- -//! the files hold the wrong columns for where they end up attached. - -use std::sync::Arc; - -use arrow_array::{Int32Array, RecordBatch}; -use arrow_schema::{DataType, Field, Schema}; -use lance::Dataset; -use lance::dataset::{CommitBuilder, InsertBuilder, WriteParams}; -use lance_table::format::DataFile; -use lance_table::transaction::action::{ - Action, AddDataFile, AddField, AddFragment, CompositeOperation, DropField, Ref, - TombstoneFieldData, UserAction, -}; -use lance_table::transaction::{Operation, Transaction}; - -/// A two-fragment dataset, so its two data files can stand in for the files an -/// action set would otherwise have had to write. -async fn test_dataset(enable_stable_row_ids: bool) -> Dataset { - let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); - let data = - RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from_iter_values(0..10))]).unwrap(); - - InsertBuilder::new("memory://") - .with_params(&WriteParams { - enable_stable_row_ids, - max_rows_per_file: 5, - ..Default::default() - }) - .execute(vec![data]) - .await - .unwrap() -} - -fn existing_data_file(dataset: &Dataset, fragment: usize) -> DataFile { - dataset.fragments()[fragment].files[0].clone() -} - -async fn commit(dataset: Dataset, actions: Vec) -> Dataset { - let read_version = dataset.version().version; - CommitBuilder::new(Arc::new(dataset)) - .execute(Transaction::new( - read_version, - Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( - "step", actions, - )])), - None, - )) - .await - .unwrap() -} - -#[tokio::test] -async fn test_one_commit_adds_a_fragment_and_then_modifies_it() { - let dataset = test_dataset(false).await; - let before = dataset.version().version; - let first_file = existing_data_file(&dataset, 0); - let second_file = existing_data_file(&dataset, 1); - let second_path = second_file.path.clone(); - - let dataset = commit( - dataset, - vec![ - Action::AddFragment(AddFragment { - local: 0, - physical_rows: 4, - row_id_meta: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - data_change: true, - }), - Action::AddDataFile(AddDataFile { - fragment: Ref::Local(0), - file: first_file, - field_ids: vec![Ref::Committed(0)], - data_change: true, - }), - // The same commit then replaces the data it just added, naming the - // fragment by the token it was minted under. - Action::TombstoneFieldData(TombstoneFieldData { - fragment: Ref::Local(0), - field_ids: vec![0], - data_change: true, - }), - Action::AddDataFile(AddDataFile { - fragment: Ref::Local(0), - file: second_file, - field_ids: vec![Ref::Committed(0)], - data_change: true, - }), - ], - ) - .await; - - assert_eq!(dataset.version().version, before + 1); - let fragments = dataset.fragments(); - assert_eq!(fragments.len(), 3); - let added = fragments.last().unwrap(); - assert_eq!(added.physical_rows, Some(4)); - // The tombstoned file is gone; only the replacement survives the commit. - let paths = added - .files - .iter() - .map(|file| file.path.clone()) - .collect::>(); - assert_eq!(paths, vec![second_path]); -} - -#[tokio::test] -async fn test_one_commit_adds_a_field_and_then_fills_it() { - let dataset = test_dataset(false).await; - let fragment_id = dataset.fragments()[0].id; - let file = existing_data_file(&dataset, 1); - let path = file.path.clone(); - - let dataset = commit( - dataset, - vec![ - Action::AddField(AddField { - local: 0, - parent: None, - def: lance_core::datatypes::Field::try_from(Field::new("b", DataType::Int32, true)) - .unwrap(), - }), - Action::AddDataFile(AddDataFile { - fragment: Ref::Committed(fragment_id), - file, - field_ids: vec![Ref::Local(0)], - data_change: true, - }), - ], - ) - .await; - - let field = dataset.schema().field("b").expect("field b was added"); - let fragment = &dataset.fragments()[0]; - let added = fragment - .files - .iter() - .find(|file| file.path == path) - .expect("the new field's data file was attached"); - // The file points at the id the commit minted, which the caller never knew. - assert_eq!(added.fields.as_ref(), &[field.id]); -} - -#[tokio::test] -async fn test_one_commit_swaps_a_field_for_a_new_one() { - let dataset = test_dataset(false).await; - let fragment_id = dataset.fragments()[0].id; - let file = existing_data_file(&dataset, 1); - - let dataset = commit( - dataset, - vec![ - Action::DropField(DropField { field: 0 }), - Action::AddField(AddField { - local: 0, - parent: None, - def: lance_core::datatypes::Field::try_from(Field::new("a", DataType::Int64, true)) - .unwrap(), - }), - Action::AddDataFile(AddDataFile { - fragment: Ref::Committed(fragment_id), - file, - field_ids: vec![Ref::Local(0)], - data_change: true, - }), - ], - ) - .await; - - // Dropping "a" and adding a new "a" of a different type is one version, and - // the new field gets a fresh id rather than inheriting the dropped one. - let schema = dataset.schema(); - assert_eq!(schema.fields.len(), 1); - let field = schema.field("a").unwrap(); - assert_ne!(field.id, 0); - assert_eq!(field.logical_type.to_string(), "int64"); -} - -#[tokio::test] -async fn test_one_commit_assigns_row_ids_to_the_fragments_it_mints() { - let dataset = test_dataset(true).await; - let next_row_id = dataset.manifest().next_row_id; - assert_eq!(next_row_id, 10); - - let dataset = commit( - dataset, - vec![ - Action::AddFragment(AddFragment { - local: 0, - physical_rows: 4, - row_id_meta: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - data_change: true, - }), - Action::AddFragment(AddFragment { - local: 1, - physical_rows: 6, - row_id_meta: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - data_change: true, - }), - ], - ) - .await; - - assert_eq!(dataset.manifest().next_row_id, 20); - for fragment in dataset.fragments().iter().skip(2) { - assert!( - fragment.row_id_meta.is_some(), - "fragment {} was minted without row ids", - fragment.id - ); - assert!(fragment.created_at_version_meta.is_some()); - } -} - -#[tokio::test] -async fn test_two_action_sets_on_disjoint_coordinates_both_commit() { - let dataset = Arc::new(test_dataset(false).await); - let read_version = dataset.version().version; - - let append = |local| { - vec![Action::AddFragment(AddFragment { - local, - physical_rows: 4, - row_id_meta: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - data_change: true, - })] - }; - - let first = CommitBuilder::new(dataset.clone()) - .execute(Transaction::new( - read_version, - Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( - "step", - append(0), - )])), - None, - )) - .await - .unwrap(); - - // The second commit still reads the original version, so it has to be - // checked against the first. Both only mint, so neither writes anything - // the other does. - let second = CommitBuilder::new(dataset) - .execute(Transaction::new( - read_version, - Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( - "step", - append(0), - )])), - None, - )) - .await - .unwrap(); - - assert_eq!(second.version().version, first.version().version + 1); - assert_eq!(second.fragments().len(), 4); -} - -#[tokio::test] -async fn test_two_action_sets_writing_the_same_field_data_conflict() { - let dataset = Arc::new(test_dataset(false).await); - let read_version = dataset.version().version; - let fragment_id = dataset.fragments()[0].id; - - let tombstone = || { - Transaction::new( - read_version, - Operation::CompositeOperation(CompositeOperation::new(vec![UserAction::new( - "step", - vec![Action::TombstoneFieldData(TombstoneFieldData { - fragment: Ref::Committed(fragment_id), - field_ids: vec![0], - data_change: true, - })], - )])), - None, - ) - }; - - CommitBuilder::new(dataset.clone()) - .execute(tombstone()) - .await - .unwrap(); - - let error = CommitBuilder::new(dataset) - .with_max_retries(0) - .execute(tombstone()) - .await - .unwrap_err(); - assert!( - error.to_string().contains("preempted"), - "unexpected error: {error}" - ); -} From 64cb96d76d3ab25e54414030d03eeaa0a32b419b Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 18 Aug 2026 16:34:28 -0700 Subject: [PATCH 21/24] docs(transaction): give the real reason SetDeletionFile takes no Ref The field claimed a minted fragment "has no committed rows to delete", which reads as a semantic restriction and is at best ambiguous. The actual constraint is mechanical: a deletion file's path embeds the fragment id, so the writer must know the committed id before it can write the file, and a minted id does not exist until apply. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/transaction/action/set_deletion_file.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rust/lance-table/src/transaction/action/set_deletion_file.rs b/rust/lance-table/src/transaction/action/set_deletion_file.rs index 7434bcf819a..02eac9b28e3 100644 --- a/rust/lance-table/src/transaction/action/set_deletion_file.rs +++ b/rust/lance-table/src/transaction/action/set_deletion_file.rs @@ -18,9 +18,13 @@ use lance_core::{Error, Result}; /// derived by diffing against the read version rather than serialized. #[derive(Debug, Clone, PartialEq, DeepSizeOf)] pub struct SetDeletionFile { - /// The fragment, by committed id. Unlike its sibling fragment actions this - /// takes no [`Ref`](super::Ref): a fragment minted in the same operation has - /// no committed rows to delete. + /// The fragment, by committed id. + /// + /// Unlike its sibling fragment actions this takes no [`Ref`](super::Ref). + /// A deletion file's path is `{fragment_id}-{read_version}-{id}.{suffix}` + /// (see [`deletion_file_path`](crate::io::deletion::deletion_file_path)), + /// so the writer has to know the committed fragment id before it can write + /// the file at all, and a minted id does not exist until apply. pub fragment: u64, /// The new deletion file, or `None` to clear the fragment's deletions. pub deletion_file: Option, From 75bef6d2250700c2157c7a939e730ecc7ef3d101 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 18 Aug 2026 16:34:33 -0700 Subject: [PATCH 22/24] refactor(transaction): split applying actions from assembling the manifest `build_manifest_from_actions` ran the actions and then assembled the manifest in one 80-line body. Move the assembly to `ApplyState::into_manifest`, so the entry point reads as the three steps it is: make the state, run the actions, turn it into a manifest. `ApplyState` now borrows the manifest it was built from rather than copying everything it needs out of it, which is also what lets `into_manifest` reach the read version without being handed it back. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/transaction/action/apply.rs | 154 ++++++++++-------- 1 file changed, 85 insertions(+), 69 deletions(-) diff --git a/rust/lance-table/src/transaction/action/apply.rs b/rust/lance-table/src/transaction/action/apply.rs index 0d080de8a35..7f7a4726baa 100644 --- a/rust/lance-table/src/transaction/action/apply.rs +++ b/rust/lance-table/src/transaction/action/apply.rs @@ -52,83 +52,20 @@ impl Transaction { )); } - let new_version = current_manifest.version + 1; let mut state = ApplyState::new(current_manifest); for action in composite_operation.iter_actions() { action.apply(&mut state)?; } - let mut next_row_id = current_manifest - .uses_stable_row_ids() - .then_some(current_manifest.next_row_id); - state.assign_row_ids_to_minted_fragments(&mut next_row_id, new_version)?; - - let ApplyState { - schema, - mut fragments, - new_bases, - rebound_fields, - reserved_fragment_ids, - reset, - config: dataset_config, - table_metadata, - .. - } = state; - - let mut indices = current_indices; - if reset { - indices.clear(); - } - prune_rebound_fields_from_indices(&mut indices, &rebound_fields); - Self::retain_relevant_indices(&mut indices, &schema, &fragments); - - Self::normalize_fragments(&mut fragments)?; - let mut manifest = self.assemble_manifest( - Some(current_manifest), - schema, - fragments, - HashMap::new(), - false, - config, - )?; - - for base in new_bases { - manifest.base_paths.insert(base.id, base); - } - - manifest.config = dataset_config; - manifest.table_metadata = table_metadata; - - // A reserved id backs no fragment, so the manifest assembly cannot - // derive it from the fragment list; raise the high-water mark to cover - // the range so a later writer's ids are not handed out twice. - if let Some(high_water) = reserved_fragment_ids { - let high_water = u32::try_from(high_water).map_err(|_| { - Error::invalid_input(format!( - "reserving fragment ids up to {high_water} exceeds the maximum fragment id \ - ({})", - u32::MAX - )) - })?; - manifest.max_fragment_id = Some( - manifest - .max_fragment_id - .map_or(high_water, |current| current.max(high_water)), - ); - } - - manifest.transaction_file = Some(transaction_file_path.to_string()); - if let Some(next_row_id) = next_row_id { - manifest.next_row_id = next_row_id; - } - - Ok((manifest, indices)) + state.into_manifest(self, current_indices, transaction_file_path, config) } } /// The read-version state an action set is applied against, plus the id /// allocations made so far. -pub(super) struct ApplyState { +pub(super) struct ApplyState<'a> { + /// The read version this delta applies to. + current_manifest: &'a Manifest, schema: Schema, fragments: Vec, /// Base paths minted by this operation. Kept apart from the manifest's own @@ -167,9 +104,10 @@ pub(super) struct ApplyState { reset: bool, } -impl ApplyState { - fn new(manifest: &Manifest) -> Self { +impl<'a> ApplyState<'a> { + fn new(manifest: &'a Manifest) -> Self { Self { + current_manifest: manifest, schema: manifest.schema.clone(), fragments: manifest.fragments.as_ref().clone(), new_bases: Vec::new(), @@ -194,6 +132,84 @@ impl ApplyState { } } + /// Assemble the next manifest from the state the actions left behind. + fn into_manifest( + mut self, + transaction: &Transaction, + current_indices: Vec, + transaction_file_path: &str, + config: &ManifestBuildConfig, + ) -> Result<(Manifest, Vec)> { + let current_manifest = self.current_manifest; + let new_version = current_manifest.version + 1; + + let mut next_row_id = current_manifest + .uses_stable_row_ids() + .then_some(current_manifest.next_row_id); + self.assign_row_ids_to_minted_fragments(&mut next_row_id, new_version)?; + + let ApplyState { + schema, + mut fragments, + new_bases, + rebound_fields, + reserved_fragment_ids, + reset, + config: dataset_config, + table_metadata, + .. + } = self; + + let mut indices = current_indices; + if reset { + indices.clear(); + } + prune_rebound_fields_from_indices(&mut indices, &rebound_fields); + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + Transaction::normalize_fragments(&mut fragments)?; + let mut manifest = transaction.assemble_manifest( + Some(current_manifest), + schema, + fragments, + HashMap::new(), + false, + config, + )?; + + for base in new_bases { + manifest.base_paths.insert(base.id, base); + } + + manifest.config = dataset_config; + manifest.table_metadata = table_metadata; + + // A reserved id backs no fragment, so the manifest assembly cannot + // derive it from the fragment list; raise the high-water mark to cover + // the range so a later writer's ids are not handed out twice. + if let Some(high_water) = reserved_fragment_ids { + let high_water = u32::try_from(high_water).map_err(|_| { + Error::invalid_input(format!( + "reserving fragment ids up to {high_water} exceeds the maximum fragment id \ + ({})", + u32::MAX + )) + })?; + manifest.max_fragment_id = Some( + manifest + .max_fragment_id + .map_or(high_water, |current| current.max(high_water)), + ); + } + + manifest.transaction_file = Some(transaction_file_path.to_string()); + if let Some(next_row_id) = next_row_id { + manifest.next_row_id = next_row_id; + } + + Ok((manifest, indices)) + } + pub(super) fn schema(&self) -> &Schema { &self.schema } From 2aaa1ec2ecf2c6efe5b01853fe60ac995978cce5 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 18 Aug 2026 16:34:45 -0700 Subject: [PATCH 23/24] feat(transaction): reference fields by Ref in the field actions `AlterField`, `DropField`, and `TombstoneFieldData` took raw committed field ids, so none of them could name a field minted earlier in the same operation. Squashing needs exactly that: collapsing "add column c" and "rename c" into one operation leaves an `AddField` whose field has no committed id, and a squash rewrites already-committed transactions, so it cannot re-plan the data to avoid the reference. All three now take a `Ref`, matching the fragment actions. Footprints follow the rule already used there: a local reference records no coordinate, since a field this operation mints is invisible to a concurrent writer. `AlterField` loses its `Default` impl -- `Ref` has no meaningful zero, and a default field reference would silently mean field 0. Co-Authored-By: Claude Opus 5 (1M context) --- protos/transaction/actions.proto | 8 +- rust/lance-table/src/transaction/action.rs | 5 +- .../src/transaction/action/add_data_file.rs | 7 +- .../src/transaction/action/alter_field.rs | 60 ++++++++++---- .../src/transaction/action/config_update.rs | 8 +- .../src/transaction/action/drop_field.rs | 83 +++++++++++++++---- .../src/transaction/action/footprint.rs | 50 +++++++---- .../src/transaction/action/proto.rs | 27 ++---- .../action/tombstone_field_data.rs | 67 ++++++++++++--- .../src/transaction/action/translate.rs | 2 +- .../src/dataset/tests/dataset_transactions.rs | 8 +- rust/lance/src/io/commit/conflict_resolver.rs | 4 +- 12 files changed, 230 insertions(+), 99 deletions(-) diff --git a/protos/transaction/actions.proto b/protos/transaction/actions.proto index 823c5d840d5..3bc5e580c7e 100644 --- a/protos/transaction/actions.proto +++ b/protos/transaction/actions.proto @@ -284,8 +284,8 @@ message AddBase { */ message TombstoneFieldData { Ref fragment = 1; - // Committed field ids whose current backing is tombstoned. - repeated uint64 field_ids = 2; + // The fields whose current backing is tombstoned. + repeated Ref field_ids = 2; // Data-change marker; see AddFragment.data_change. optional bool data_change = 3; } @@ -374,7 +374,7 @@ message UpdateCompactedSsTables { * no live fields). References an existing committed field id. */ message DropField { - uint64 field = 1; + Ref field = 1; } /* @@ -387,7 +387,7 @@ message DropField { * AddDataFile to rewrite the data. New facets are added as optional fields. */ message AlterField { - uint64 field = 1; + Ref field = 1; optional string name = 2; // New Arrow logical type (see Field.logical_type). The cast. optional string logical_type = 3; diff --git a/rust/lance-table/src/transaction/action.rs b/rust/lance-table/src/transaction/action.rs index 703adb77db0..0aa301d36c2 100644 --- a/rust/lance-table/src/transaction/action.rs +++ b/rust/lance-table/src/transaction/action.rs @@ -258,9 +258,10 @@ mod tests { #[test] fn test_data_change_defaults_by_action_kind() { let alter = Action::AlterField(AlterField { - field: 1, + field: Ref::Committed(1), name: Some("renamed".into()), - ..Default::default() + logical_type: None, + nullable: None, }); assert!(!alter.is_data_change(), "a rename changes no row values"); diff --git a/rust/lance-table/src/transaction/action/add_data_file.rs b/rust/lance-table/src/transaction/action/add_data_file.rs index 59c673db0b2..10d59c627d5 100644 --- a/rust/lance-table/src/transaction/action/add_data_file.rs +++ b/rust/lance-table/src/transaction/action/add_data_file.rs @@ -59,12 +59,7 @@ impl AddDataFile { /// The data of every committed field the file backs, in the fragment it is /// attached to. A file backing only minted fields writes nothing. pub(super) fn footprint(&self, footprint: &mut Footprint) { - footprint.add_field_data( - self.fragment, - self.field_ids - .iter() - .filter_map(|field| field.committed().and_then(|id| i32::try_from(id).ok())), - ); + footprint.add_field_data(self.fragment, self.field_ids.iter().copied()); } } diff --git a/rust/lance-table/src/transaction/action/alter_field.rs b/rust/lance-table/src/transaction/action/alter_field.rs index e3380056b87..4c475ae7735 100644 --- a/rust/lance-table/src/transaction/action/alter_field.rs +++ b/rust/lance-table/src/transaction/action/alter_field.rs @@ -4,8 +4,8 @@ //! Alter facets of an existing field in place. use super::apply::ApplyState; -use super::proto::field_id_from_wire; -use super::{Coordinate, Footprint}; +use super::proto::required; +use super::{Footprint, Ref}; use crate::format::pb; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result}; @@ -17,9 +17,9 @@ use lance_core::{Error, Result}; /// the same field commute. A cast additionally needs a /// [`TombstoneFieldData`](super::TombstoneFieldData) plus a fresh /// [`AddDataFile`](super::AddDataFile) to rewrite the data. -#[derive(Debug, Clone, PartialEq, DeepSizeOf, Default)] +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] pub struct AlterField { - pub field: i32, + pub field: Ref, pub name: Option, /// The new Arrow logical type. The cast. pub logical_type: Option, @@ -28,13 +28,13 @@ pub struct AlterField { impl AlterField { pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { + let field_id = state.resolve_field(self.field)?; let field = state .schema_mut() - .field_by_id_mut(self.field) + .field_by_id_mut(field_id) .ok_or_else(|| { Error::invalid_input(format!( - "AlterField names field {}, which does not exist", - self.field + "AlterField names field {field_id}, which does not exist" )) })?; if let Some(name) = &self.name { @@ -48,7 +48,7 @@ impl AlterField { // The cast leaves any index on the field describing the old type. // The data rewrite itself is separate actions; this only records // that every fragment's view of the field changed. - state.rebind_field_everywhere(self.field); + state.rebind_field_everywhere(field_id); } Ok(()) } @@ -62,14 +62,14 @@ impl AlterField { /// The field's definition. The data rewrite a cast needs is separate /// actions, which record their own coordinates. pub(super) fn footprint(&self, footprint: &mut Footprint) { - footprint.add(Coordinate::FieldDefinition(self.field)); + footprint.add_field_definition(self.field); } } impl From<&AlterField> for pb::AlterField { fn from(value: &AlterField) -> Self { Self { - field: value.field as u64, + field: Some(value.field.into()), name: value.name.clone(), logical_type: value.logical_type.clone(), nullable: value.nullable, @@ -82,7 +82,7 @@ impl TryFrom for AlterField { fn try_from(message: pb::AlterField) -> Result { Ok(Self { - field: field_id_from_wire(message.field)?, + field: required(message.field, "AlterField.field")?.try_into()?, name: message.name, logical_type: message.logical_type, nullable: message.nullable, @@ -102,7 +102,7 @@ mod tests { let (next, indices) = apply_with_indices( &backed_manifest(), vec![Action::AlterField(AlterField { - field: 0, + field: Ref::Committed(0), name: Some("renamed".into()), logical_type: None, nullable: Some(true), @@ -123,7 +123,7 @@ mod tests { let (next, indices) = apply_with_indices( &backed_manifest(), vec![Action::AlterField(AlterField { - field: 0, + field: Ref::Committed(0), name: None, logical_type: Some("int64".into()), nullable: None, @@ -144,9 +144,10 @@ mod tests { let error = apply( &backed_manifest(), vec![Action::AlterField(AlterField { - field: 7, + field: Ref::Committed(7), name: Some("nope".into()), - ..Default::default() + logical_type: None, + nullable: None, })], ) .unwrap_err(); @@ -154,4 +155,33 @@ mod tests { assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); assert!(error.to_string().contains("field 7"), "{error}"); } + + #[test] + fn test_alter_field_can_name_a_field_minted_in_the_same_operation() { + // A squash of "add column" and "rename column" lowers to exactly this, + // and has no committed id to name the field by. + use crate::transaction::action::AddField; + use crate::transaction::action::test_support::added_field; + + let next = apply( + &backed_manifest(), + vec![ + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("before"), + }), + Action::AlterField(AlterField { + field: Ref::Local(0), + name: Some("after".into()), + logical_type: None, + nullable: None, + }), + ], + ) + .unwrap(); + + assert!(next.schema.field("before").is_none()); + assert!(next.schema.field("after").is_some()); + } } diff --git a/rust/lance-table/src/transaction/action/config_update.rs b/rust/lance-table/src/transaction/action/config_update.rs index ac1200dbb61..9ef57f506ad 100644 --- a/rust/lance-table/src/transaction/action/config_update.rs +++ b/rust/lance-table/src/transaction/action/config_update.rs @@ -482,8 +482,12 @@ mod tests { }], ..Default::default() })]); - let dropped = footprint(vec![Action::DropField(DropField { field: 1 })]); - let other = footprint(vec![Action::DropField(DropField { field: 2 })]); + let dropped = footprint(vec![Action::DropField(DropField { + field: Ref::Committed(1), + })]); + let other = footprint(vec![Action::DropField(DropField { + field: Ref::Committed(2), + })]); assert!(dropped.conflicts_with(&metadata)); assert!(metadata.conflicts_with(&dropped)); diff --git a/rust/lance-table/src/transaction/action/drop_field.rs b/rust/lance-table/src/transaction/action/drop_field.rs index b4b5ceba8c2..fdec98364cf 100644 --- a/rust/lance-table/src/transaction/action/drop_field.rs +++ b/rust/lance-table/src/transaction/action/drop_field.rs @@ -3,9 +3,9 @@ //! Remove a field from the schema. -use super::Footprint; use super::apply::{ApplyState, TOMBSTONED_FIELD}; -use super::proto::field_id_from_wire; +use super::proto::required; +use super::{Footprint, Ref}; use crate::format::pb; use lance_core::datatypes::Field; use lance_core::deepsize::DeepSizeOf; @@ -19,21 +19,21 @@ use std::collections::HashSet; /// index over a removed field is discarded. #[derive(Debug, Clone, PartialEq, DeepSizeOf)] pub struct DropField { - pub field: i32, + pub field: Ref, } impl DropField { pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { - let field = state.schema().field_by_id(self.field).ok_or_else(|| { + let field_id = state.resolve_field(self.field)?; + let field = state.schema().field_by_id(field_id).ok_or_else(|| { Error::invalid_input(format!( - "DropField names field {}, which does not exist", - self.field + "DropField names field {field_id}, which does not exist" )) })?; // A struct's children cannot outlive it, so the whole subtree goes. let mut dropped = HashSet::new(); collect_subtree_ids(field, &mut dropped); - remove_field(&mut state.schema_mut().fields, self.field); + remove_field(&mut state.schema_mut().fields, field_id); // The fields are gone from the schema, so the slots that backed them in // each data file are dead. Tombstoning rather than rewriting the field @@ -105,7 +105,7 @@ fn remove_field(fields: &mut Vec, field_id: i32) { impl From<&DropField> for pb::DropField { fn from(value: &DropField) -> Self { Self { - field: value.field as u64, + field: Some(value.field.into()), } } } @@ -115,7 +115,7 @@ impl TryFrom for DropField { fn try_from(message: pb::DropField) -> Result { Ok(Self { - field: field_id_from_wire(message.field)?, + field: required(message.field, "DropField.field")?.try_into()?, }) } } @@ -136,7 +136,9 @@ mod tests { fn test_drop_field_removes_it_and_its_data() { let (next, indices) = apply_with_indices( &backed_manifest(), - vec![Action::DropField(DropField { field: 0 })], + vec![Action::DropField(DropField { + field: Ref::Committed(0), + })], vec![sample_index_metadata("idx")], ) .unwrap(); @@ -158,7 +160,13 @@ mod tests { fragment.files[0] = DataFile::new("data/0.lance", vec![0, 1], vec![0, 1], 2, 0, None, None); manifest.fragments = Arc::new(vec![fragment]); - let next = apply(&manifest, vec![Action::DropField(DropField { field: 0 })]).unwrap(); + let next = apply( + &manifest, + vec![Action::DropField(DropField { + field: Ref::Committed(0), + })], + ) + .unwrap(); assert!(next.schema.field_by_id(0).is_none()); assert!(next.schema.field_by_id(1).is_some()); @@ -180,7 +188,13 @@ mod tests { parent.children.push(child); manifest.schema.fields.push(parent); - let next = apply(&manifest, vec![Action::DropField(DropField { field: 1 })]).unwrap(); + let next = apply( + &manifest, + vec![Action::DropField(DropField { + field: Ref::Committed(1), + })], + ) + .unwrap(); assert!(next.schema.field_by_id(1).is_none()); assert!( @@ -193,7 +207,9 @@ mod tests { fn test_drop_field_rejects_a_missing_field() { let error = apply( &backed_manifest(), - vec![Action::DropField(DropField { field: 7 })], + vec![Action::DropField(DropField { + field: Ref::Committed(7), + })], ) .unwrap_err(); @@ -206,7 +222,9 @@ mod tests { let next = apply( &backed_manifest(), vec![ - Action::DropField(DropField { field: 0 }), + Action::DropField(DropField { + field: Ref::Committed(0), + }), Action::AddField(AddField { local: 0, parent: None, @@ -225,4 +243,41 @@ mod tests { .collect::>(); assert_eq!(ids, vec![1]); } + + #[test] + fn test_drop_field_rejects_a_field_id_out_of_range() { + let error = apply( + &backed_manifest(), + vec![Action::DropField(DropField { + field: Ref::Committed(u64::from(u32::MAX) + 1), + })], + ) + .unwrap_err(); + assert!( + error.to_string().contains("out of range"), + "unexpected message: {error}" + ); + } + + #[test] + fn test_drop_field_can_name_a_field_minted_in_the_same_operation() { + // A squash of "add column" and "drop column" lowers to exactly this, + // and has no committed id to name the field by. + let next = apply( + &backed_manifest(), + vec![ + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("transient"), + }), + Action::DropField(DropField { + field: Ref::Local(0), + }), + ], + ) + .unwrap(); + + assert!(next.schema.field("transient").is_none()); + } } diff --git a/rust/lance-table/src/transaction/action/footprint.rs b/rust/lance-table/src/transaction/action/footprint.rs index 8b8285e2f64..c039b49db37 100644 --- a/rust/lance-table/src/transaction/action/footprint.rs +++ b/rust/lance-table/src/transaction/action/footprint.rs @@ -163,17 +163,25 @@ impl Footprint { self.writes.insert(coordinate); } - /// The data of each field within `fragment`. A fragment minted in this same - /// operation records nothing: no concurrent writer can be naming it. - pub(super) fn add_field_data(&mut self, fragment: Ref, fields: impl IntoIterator) { + /// The data of each field within `fragment`. A fragment or field minted in + /// this same operation records nothing: no concurrent writer can be naming + /// one. + pub(super) fn add_field_data(&mut self, fragment: Ref, fields: impl IntoIterator) { let Some(fragment) = fragment.committed() else { return; }; - for field in fields { + for field in fields.into_iter().filter_map(committed_field) { self.add(Coordinate::FieldData { fragment, field }); } } + /// A field's entry in the schema. + pub(super) fn add_field_definition(&mut self, field: Ref) { + if let Some(field) = committed_field(field) { + self.add(Coordinate::FieldDefinition(field)); + } + } + pub(super) fn remove_fragment(&mut self, fragment: u64) { self.add(Coordinate::FragmentExistence(fragment)); self.removed_fragments.insert(fragment); @@ -200,12 +208,20 @@ impl Footprint { self.exclusive = true; } - pub(super) fn remove_field(&mut self, field: i32) { - self.add(Coordinate::FieldDefinition(field)); - self.removed_fields.insert(field); + pub(super) fn remove_field(&mut self, field: Ref) { + if let Some(field) = committed_field(field) { + self.add(Coordinate::FieldDefinition(field)); + self.removed_fields.insert(field); + } } } +/// A field id a concurrent writer could also be naming, or `None` for a field +/// this operation mints, which no one else can see yet. +fn committed_field(reference: Ref) -> Option { + i32::try_from(reference.committed()?).ok() +} + impl From<&CompositeOperation> for Footprint { fn from(composite_operation: &CompositeOperation) -> Self { let mut footprint = Self::default(); @@ -260,7 +276,7 @@ mod tests { fn tombstone(fragment: u64, fields: &[i32]) -> Action { Action::TombstoneFieldData(TombstoneFieldData { fragment: Ref::Committed(fragment), - field_ids: fields.to_vec(), + field_ids: fields.iter().map(|id| Ref::Committed(*id as u64)).collect(), data_change: true, }) } @@ -340,32 +356,32 @@ mod tests { false, )] #[case::same_field_definition( - vec![Action::AlterField(AlterField { field: 1, name: Some("a".into()), ..Default::default() })], - vec![Action::AlterField(AlterField { field: 1, nullable: Some(true), ..Default::default() })], + vec![Action::AlterField(AlterField { field: Ref::Committed(1), name: Some("a".into()), logical_type: None, nullable: None })], + vec![Action::AlterField(AlterField { field: Ref::Committed(1), name: None, logical_type: None, nullable: Some(true) })], true, )] #[case::different_field_definitions( - vec![Action::AlterField(AlterField { field: 1, ..Default::default() })], - vec![Action::AlterField(AlterField { field: 2, ..Default::default() })], + vec![Action::AlterField(AlterField { field: Ref::Committed(1), name: None, logical_type: None, nullable: None })], + vec![Action::AlterField(AlterField { field: Ref::Committed(2), name: None, logical_type: None, nullable: None })], false, )] #[case::dropping_a_field_collides_with_altering_it( - vec![Action::DropField(DropField { field: 1 })], - vec![Action::AlterField(AlterField { field: 1, nullable: Some(true), ..Default::default() })], + vec![Action::DropField(DropField { field: Ref::Committed(1) })], + vec![Action::AlterField(AlterField { field: Ref::Committed(1), name: None, logical_type: None, nullable: Some(true) })], true, )] #[case::dropping_a_field_collides_with_rewriting_its_data( - vec![Action::DropField(DropField { field: 1 })], + vec![Action::DropField(DropField { field: Ref::Committed(1) })], vec![tombstone(0, &[1])], true, )] #[case::dropping_a_field_leaves_other_fields_alone( - vec![Action::DropField(DropField { field: 1 })], + vec![Action::DropField(DropField { field: Ref::Committed(1) })], vec![tombstone(0, &[2])], false, )] #[case::dropping_a_field_leaves_deletions_alone( - vec![Action::DropField(DropField { field: 1 })], + vec![Action::DropField(DropField { field: Ref::Committed(1) })], vec![set_deletion_file(0)], false, )] diff --git a/rust/lance-table/src/transaction/action/proto.rs b/rust/lance-table/src/transaction/action/proto.rs index 6cb50b8a3e3..196925a82fe 100644 --- a/rust/lance-table/src/transaction/action/proto.rs +++ b/rust/lance-table/src/transaction/action/proto.rs @@ -16,16 +16,6 @@ use super::{Action, CompositeOperation, Ref, UserAction}; use crate::format::pb; use lance_core::{Error, Result}; -/// A field id on the wire is a `uint64`; in the manifest it is an `i32`. -pub(super) fn field_id_from_wire(id: u64) -> Result { - i32::try_from(id).map_err(|_| { - Error::invalid_input(format!( - "field id {id} in an action exceeds the maximum field id ({})", - i32::MAX - )) - }) -} - /// `data_change` is absent-means-true on the wire, so only the `false` case is /// written out. pub(super) fn data_change_to_wire(data_change: bool) -> Option { @@ -201,7 +191,7 @@ mod tests { }), Action::TombstoneFieldData(TombstoneFieldData { fragment: Ref::Committed(4), - field_ids: vec![7, 8], + field_ids: vec![Ref::Committed(7), Ref::Committed(8)], data_change: true, }), Action::RemoveFragment(RemoveFragment { @@ -220,12 +210,14 @@ mod tests { data_change: true, }), Action::AlterField(AlterField { - field: 2, + field: Ref::Committed(2), name: Some("renamed".into()), logical_type: Some("int64".into()), nullable: Some(false), }), - Action::DropField(DropField { field: 3 }), + Action::DropField(DropField { + field: Ref::Committed(3), + }), Action::ReserveFragmentIds(ReserveFragmentIds { count: 4 }), Action::ResetTable(ResetTable), Action::ConfigUpdate(ConfigUpdate { @@ -311,13 +303,4 @@ mod tests { "unexpected message: {error}" ); } - - #[test] - fn test_field_id_out_of_range_is_rejected() { - let error = field_id_from_wire(u64::from(u32::MAX) + 1).unwrap_err(); - assert!( - error.to_string().contains("exceeds the maximum field id"), - "unexpected message: {error}" - ); - } } diff --git a/rust/lance-table/src/transaction/action/tombstone_field_data.rs b/rust/lance-table/src/transaction/action/tombstone_field_data.rs index 505daab5433..5a120e8d94b 100644 --- a/rust/lance-table/src/transaction/action/tombstone_field_data.rs +++ b/rust/lance-table/src/transaction/action/tombstone_field_data.rs @@ -4,7 +4,7 @@ //! Tombstone the data-file binding of committed fields within one fragment. use super::apply::{ApplyState, TOMBSTONED_FIELD}; -use super::proto::{data_change_from_wire, data_change_to_wire, field_id_from_wire, required}; +use super::proto::{data_change_from_wire, data_change_to_wire, required}; use super::{Footprint, Ref}; use crate::format::pb; use lance_core::deepsize::DeepSizeOf; @@ -20,8 +20,8 @@ use lance_core::{Error, Result}; #[derive(Debug, Clone, PartialEq, DeepSizeOf)] pub struct TombstoneFieldData { pub fragment: Ref, - /// Committed field ids whose current backing is tombstoned. - pub field_ids: Vec, + /// The fields whose current backing is tombstoned. + pub field_ids: Vec, /// `false` marks a tombstone whose data is re-added by an /// [`AddDataFile`](super::AddDataFile) for the same fields in the same /// operation, i.e. a re-encode that moves the bytes without changing any @@ -33,9 +33,14 @@ pub struct TombstoneFieldData { impl TombstoneFieldData { pub(super) fn apply(&self, state: &mut ApplyState) -> Result<()> { let fragment_id = state.resolve_fragment(self.fragment)?; + let field_ids = self + .field_ids + .iter() + .map(|field| state.resolve_field(*field)) + .collect::>>()?; let fragment = state.fragment_mut(fragment_id, "TombstoneFieldData")?; - for &field_id in &self.field_ids { + for &field_id in &field_ids { let mut found = false; for file in fragment.files.iter_mut() { let Some(position) = file.fields.iter().position(|id| *id == field_id) else { @@ -56,14 +61,13 @@ impl TombstoneFieldData { // New values for these fields supersede any overlay still shadowing // them, so the drop is not silently masked by stale overlay cells. - let overlaid: Vec = self - .field_ids + let overlaid: Vec = field_ids .iter() .filter_map(|id| u32::try_from(*id).ok()) .collect(); crate::format::overlay::tombstone_overlay_fields(&mut fragment.overlays, &overlaid); - state.rebind_fields(fragment_id, self.field_ids.iter().copied()); + state.rebind_fields(fragment_id, field_ids.iter().copied()); Ok(()) } @@ -82,7 +86,7 @@ impl From<&TombstoneFieldData> for pb::TombstoneFieldData { fn from(value: &TombstoneFieldData) -> Self { Self { fragment: Some(value.fragment.into()), - field_ids: value.field_ids.iter().map(|id| *id as u64).collect(), + field_ids: value.field_ids.iter().map(|id| (*id).into()).collect(), data_change: data_change_to_wire(value.data_change), } } @@ -97,7 +101,7 @@ impl TryFrom for TombstoneFieldData { field_ids: message .field_ids .into_iter() - .map(field_id_from_wire) + .map(Ref::try_from) .collect::>>()?, data_change: data_change_from_wire(message.data_change), }) @@ -116,7 +120,7 @@ mod tests { fn tombstone_field_zero() -> Action { Action::TombstoneFieldData(TombstoneFieldData { fragment: Ref::Committed(0), - field_ids: vec![0], + field_ids: vec![Ref::Committed(0)], data_change: true, }) } @@ -163,7 +167,7 @@ mod tests { &backed_manifest(), vec![Action::TombstoneFieldData(TombstoneFieldData { fragment: Ref::Committed(0), - field_ids: vec![7], + field_ids: vec![Ref::Committed(7)], data_change: true, })], ) @@ -172,4 +176,45 @@ mod tests { assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); assert!(error.to_string().contains("field 7"), "{error}"); } + + #[test] + fn test_tombstone_field_data_can_name_a_field_minted_in_the_same_operation() { + // A squash of "add column with data" and "re-encode that column" lowers + // to exactly this, and has no committed id to name the field by. + use crate::transaction::action::test_support::added_field; + use crate::transaction::action::{AddDataFile, AddField}; + + let next = apply( + &backed_manifest(), + vec![ + Action::AddField(AddField { + local: 0, + parent: None, + def: added_field("fresh"), + }), + Action::AddDataFile(AddDataFile { + fragment: Ref::Committed(0), + file: DataFile::new_unstarted("data/fresh.lance", 1, 0), + field_ids: vec![Ref::Local(0)], + data_change: true, + }), + Action::TombstoneFieldData(TombstoneFieldData { + fragment: Ref::Committed(0), + field_ids: vec![Ref::Local(0)], + data_change: false, + }), + ], + ) + .unwrap(); + + // The file that backed only the minted field is left backing nothing + // and is pruned, so the fragment keeps just its original file. + assert_eq!(next.fragments[0].files.len(), 1); + assert!( + !next.fragments[0] + .files + .iter() + .any(|file| file.path == "data/fresh.lance") + ); + } } diff --git a/rust/lance-table/src/transaction/action/translate.rs b/rust/lance-table/src/transaction/action/translate.rs index 1e7f2d076e9..271ba6eba29 100644 --- a/rust/lance-table/src/transaction/action/translate.rs +++ b/rust/lance-table/src/transaction/action/translate.rs @@ -141,7 +141,7 @@ fn data_replacement_actions(replacements: &[DataReplacementGroup]) -> Result Transaction { + fn tombstone_txn(fragment: u64, field: u64) -> Transaction { action_txn(vec![TxnAction::TombstoneFieldData(TombstoneFieldData { fragment: ActionRef::Committed(fragment), - field_ids: vec![field], + field_ids: vec![ActionRef::Committed(field)], data_change: true, })]) } From 25f0610c983b94f74328552dd854cfabf9236512 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 19 Aug 2026 12:48:40 -0700 Subject: [PATCH 24/24] fix(transaction): adapt the action path to upstream API changes Rebasing onto the merged transaction module split brings three API changes the action code predates: `RowIdMeta::Inline` wraps `InlineRowIds` rather than a byte vector, `DataFile::new`/`new_unstarted` take a `ConcreteFileVersion` instead of a major/minor pair, and `Operation::Project` carries `preserves_nullability`. Co-Authored-By: Claude Opus 5 (1M context) --- .../src/transaction/action/add_data_file.rs | 7 ++-- .../src/transaction/action/add_field.rs | 3 +- .../src/transaction/action/add_fragment.rs | 9 +++-- .../src/transaction/action/apply.rs | 3 +- .../src/transaction/action/drop_field.rs | 10 ++++- .../src/transaction/action/footprint.rs | 3 +- .../src/transaction/action/proto.rs | 5 ++- .../src/transaction/action/reset_table.rs | 3 +- .../src/transaction/action/test_support.rs | 4 +- .../action/tombstone_field_data.rs | 12 +++++- .../src/transaction/action/translate.rs | 39 ++++++++++++++----- .../src/transaction/manifest_build.rs | 1 - rust/lance/src/io/commit/conflict_resolver.rs | 1 + 13 files changed, 72 insertions(+), 28 deletions(-) diff --git a/rust/lance-table/src/transaction/action/add_data_file.rs b/rust/lance-table/src/transaction/action/add_data_file.rs index 10d59c627d5..6fe28c7270c 100644 --- a/rust/lance-table/src/transaction/action/add_data_file.rs +++ b/rust/lance-table/src/transaction/action/add_data_file.rs @@ -97,6 +97,7 @@ mod tests { use crate::transaction::action::Action; use crate::transaction::action::test_support::apply; use crate::transaction::test_support::sample_manifest; + use lance_file::version::ConcreteFileVersion; #[test] fn test_add_data_file_rejects_an_unbound_local_token() { @@ -105,7 +106,7 @@ mod tests { &manifest, vec![Action::AddDataFile(AddDataFile { fragment: Ref::Local(3), - file: DataFile::new_unstarted("data/x.lance", 2, 0), + file: DataFile::new_unstarted("data/x.lance", ConcreteFileVersion::V2_0), field_ids: vec![Ref::Committed(0)], data_change: true, })], @@ -120,7 +121,7 @@ mod tests { #[test] fn test_add_data_file_rejects_a_column_count_mismatch() { let manifest = sample_manifest(); - let mut file = DataFile::new_unstarted("data/x.lance", 2, 0); + let mut file = DataFile::new_unstarted("data/x.lance", ConcreteFileVersion::V2_0); file.column_indices = vec![0, 1].into(); let error = apply( @@ -148,7 +149,7 @@ mod tests { &manifest, vec![Action::AddDataFile(AddDataFile { fragment: Ref::Committed(7), - file: DataFile::new_unstarted("data/x.lance", 2, 0), + file: DataFile::new_unstarted("data/x.lance", ConcreteFileVersion::V2_0), field_ids: vec![Ref::Committed(0)], data_change: true, })], diff --git a/rust/lance-table/src/transaction/action/add_field.rs b/rust/lance-table/src/transaction/action/add_field.rs index 4847edbe6d1..0fe9b6fcf24 100644 --- a/rust/lance-table/src/transaction/action/add_field.rs +++ b/rust/lance-table/src/transaction/action/add_field.rs @@ -103,6 +103,7 @@ mod tests { use crate::transaction::action::{Action, AddDataFile}; use crate::transaction::test_support::sample_manifest; use arrow_schema::{DataType, Field as ArrowField}; + use lance_file::version::ConcreteFileVersion; #[test] fn test_two_add_fields_mint_distinct_ids() { @@ -148,7 +149,7 @@ mod tests { }), Action::AddDataFile(AddDataFile { fragment: Ref::Committed(0), - file: DataFile::new_unstarted("data/added.lance", 2, 0), + file: DataFile::new_unstarted("data/added.lance", ConcreteFileVersion::V2_0), field_ids: vec![Ref::Local(7)], data_change: true, }), diff --git a/rust/lance-table/src/transaction/action/add_fragment.rs b/rust/lance-table/src/transaction/action/add_fragment.rs index 0eb3140f3c6..310a6bca259 100644 --- a/rust/lance-table/src/transaction/action/add_fragment.rs +++ b/rust/lance-table/src/transaction/action/add_fragment.rs @@ -88,7 +88,7 @@ impl From<&AddFragment> for pb::AddFragment { physical_rows: value.physical_rows, row_id_sequence: value.row_id_meta.as_ref().map(|meta| match meta { RowIdMeta::Inline(data) => { - pb::add_fragment::RowIdSequence::InlineRowIds(data.clone()) + pb::add_fragment::RowIdSequence::InlineRowIds(data.to_vec()) } RowIdMeta::External(file) => { pb::add_fragment::RowIdSequence::ExternalRowIds(external_file_to_wire(file)) @@ -137,7 +137,9 @@ impl TryFrom for AddFragment { local: message.local, physical_rows: message.physical_rows, row_id_meta: message.row_id_sequence.map(|sequence| match sequence { - pb::add_fragment::RowIdSequence::InlineRowIds(data) => RowIdMeta::Inline(data), + pb::add_fragment::RowIdSequence::InlineRowIds(data) => { + RowIdMeta::Inline(data.into()) + } pb::add_fragment::RowIdSequence::ExternalRowIds(file) => { RowIdMeta::External(external_file_from_wire(file)) } @@ -176,6 +178,7 @@ mod tests { use crate::transaction::action::test_support::apply; use crate::transaction::action::{Action, AddDataFile, Ref}; use crate::transaction::test_support::sample_manifest; + use lance_file::version::ConcreteFileVersion; #[test] fn test_add_fragment_and_data_file_mint_ids() { @@ -193,7 +196,7 @@ mod tests { }), Action::AddDataFile(AddDataFile { fragment: Ref::Local(0), - file: DataFile::new_unstarted("data/new.lance", 2, 0), + file: DataFile::new_unstarted("data/new.lance", ConcreteFileVersion::V2_0), field_ids: vec![Ref::Committed(0)], data_change: true, }), diff --git a/rust/lance-table/src/transaction/action/apply.rs b/rust/lance-table/src/transaction/action/apply.rs index 7f7a4726baa..206ef485e36 100644 --- a/rust/lance-table/src/transaction/action/apply.rs +++ b/rust/lance-table/src/transaction/action/apply.rs @@ -464,6 +464,7 @@ mod tests { use crate::transaction::action::test_support::{added_field, apply, backed_manifest}; use crate::transaction::action::{Action, AddDataFile, AddField, AddFragment}; use crate::transaction::test_support::default_build_config; + use lance_file::version::ConcreteFileVersion; #[test] fn test_an_action_set_relocates_onto_a_newer_version() { @@ -483,7 +484,7 @@ mod tests { }), Action::AddDataFile(AddDataFile { fragment: Ref::Local(0), - file: DataFile::new_unstarted("data/new.lance", 2, 0), + file: DataFile::new_unstarted("data/new.lance", ConcreteFileVersion::V2_0), field_ids: vec![Ref::Local(0)], data_change: true, }), diff --git a/rust/lance-table/src/transaction/action/drop_field.rs b/rust/lance-table/src/transaction/action/drop_field.rs index fdec98364cf..218d027cfd4 100644 --- a/rust/lance-table/src/transaction/action/drop_field.rs +++ b/rust/lance-table/src/transaction/action/drop_field.rs @@ -130,6 +130,7 @@ mod tests { use crate::transaction::action::{Action, AddField}; use crate::transaction::test_support::sample_index_metadata; use arrow_schema::{DataType, Field as ArrowField}; + use lance_file::version::ConcreteFileVersion; use std::sync::Arc; #[test] @@ -157,7 +158,14 @@ mod tests { schema_field.id = 1; manifest.schema.fields.push(schema_field); let mut fragment = manifest.fragments[0].clone(); - fragment.files[0] = DataFile::new("data/0.lance", vec![0, 1], vec![0, 1], 2, 0, None, None); + fragment.files[0] = DataFile::new( + "data/0.lance", + vec![0, 1], + vec![0, 1], + ConcreteFileVersion::V2_0, + None, + None, + ); manifest.fragments = Arc::new(vec![fragment]); let next = apply( diff --git a/rust/lance-table/src/transaction/action/footprint.rs b/rust/lance-table/src/transaction/action/footprint.rs index c039b49db37..0dbd604d950 100644 --- a/rust/lance-table/src/transaction/action/footprint.rs +++ b/rust/lance-table/src/transaction/action/footprint.rs @@ -242,6 +242,7 @@ mod tests { }; use arrow_schema::{DataType, Field as ArrowField}; use lance_core::datatypes::Field; + use lance_file::version::ConcreteFileVersion; use rstest::rstest; fn footprint(actions: Vec) -> Footprint { @@ -264,7 +265,7 @@ mod tests { fn add_data_file(fragment: Ref, fields: &[i32]) -> Action { Action::AddDataFile(AddDataFile { fragment, - file: DataFile::new_unstarted("data/x.lance", 2, 0), + file: DataFile::new_unstarted("data/x.lance", ConcreteFileVersion::V2_0), field_ids: fields .iter() .map(|field| Ref::Committed(*field as u64)) diff --git a/rust/lance-table/src/transaction/action/proto.rs b/rust/lance-table/src/transaction/action/proto.rs index 196925a82fe..c7c7f2a4b6e 100644 --- a/rust/lance-table/src/transaction/action/proto.rs +++ b/rust/lance-table/src/transaction/action/proto.rs @@ -156,10 +156,11 @@ mod tests { }; use arrow_schema::{DataType, Field as ArrowField}; use lance_core::datatypes::Field; + use lance_file::version::ConcreteFileVersion; use std::sync::Arc; fn sample_data_file() -> DataFile { - DataFile::new_unstarted("data/1.lance", 2, 0) + DataFile::new_unstarted("data/1.lance", ConcreteFileVersion::V2_0) } fn all_actions() -> Vec { @@ -167,7 +168,7 @@ mod tests { Action::AddFragment(AddFragment { local: 0, physical_rows: 10, - row_id_meta: Some(RowIdMeta::Inline(vec![1, 2, 3])), + row_id_meta: Some(RowIdMeta::Inline(vec![1, 2, 3].into())), last_updated_at_version_meta: Some(RowDatasetVersionMeta::Inline(Arc::from( [4u8, 5].as_slice(), ))), diff --git a/rust/lance-table/src/transaction/action/reset_table.rs b/rust/lance-table/src/transaction/action/reset_table.rs index 54504f94f6b..c39198d7282 100644 --- a/rust/lance-table/src/transaction/action/reset_table.rs +++ b/rust/lance-table/src/transaction/action/reset_table.rs @@ -66,6 +66,7 @@ mod tests { }; use crate::transaction::action::{Action, AddDataFile, AddField, AddFragment, Ref}; use crate::transaction::test_support::sample_index_metadata; + use lance_file::version::ConcreteFileVersion; fn reset() -> Action { Action::ResetTable(ResetTable) @@ -119,7 +120,7 @@ mod tests { }), Action::AddDataFile(AddDataFile { fragment: Ref::Local(0), - file: DataFile::new_unstarted("data/fresh.lance", 2, 0), + file: DataFile::new_unstarted("data/fresh.lance", ConcreteFileVersion::V2_0), field_ids: vec![Ref::Local(0)], data_change: true, }), diff --git a/rust/lance-table/src/transaction/action/test_support.rs b/rust/lance-table/src/transaction/action/test_support.rs index 6b72355be25..15a92932d2a 100644 --- a/rust/lance-table/src/transaction/action/test_support.rs +++ b/rust/lance-table/src/transaction/action/test_support.rs @@ -10,6 +10,7 @@ use crate::transaction::{Operation, Transaction}; use arrow_schema::{DataType, Field as ArrowField}; use lance_core::Result; use lance_core::datatypes::Field; +use lance_file::version::ConcreteFileVersion; use std::sync::Arc; pub(super) fn apply(manifest: &Manifest, actions: Vec) -> Result { @@ -45,8 +46,7 @@ pub(super) fn backed_manifest() -> Manifest { "data/0.lance", vec![0], vec![0], - 2, - 0, + ConcreteFileVersion::V2_0, None, None, )); diff --git a/rust/lance-table/src/transaction/action/tombstone_field_data.rs b/rust/lance-table/src/transaction/action/tombstone_field_data.rs index 5a120e8d94b..6cecf3ed2df 100644 --- a/rust/lance-table/src/transaction/action/tombstone_field_data.rs +++ b/rust/lance-table/src/transaction/action/tombstone_field_data.rs @@ -115,6 +115,7 @@ mod tests { use crate::transaction::action::Action; use crate::transaction::action::test_support::{apply, apply_with_indices, backed_manifest}; use crate::transaction::test_support::sample_index_metadata; + use lance_file::version::ConcreteFileVersion; use std::sync::Arc; fn tombstone_field_zero() -> Action { @@ -137,7 +138,14 @@ mod tests { fn test_tombstone_field_data_keeps_a_file_with_a_live_field() { let mut manifest = backed_manifest(); let mut fragment = manifest.fragments[0].clone(); - fragment.files[0] = DataFile::new("data/0.lance", vec![0, 1], vec![0, 1], 2, 0, None, None); + fragment.files[0] = DataFile::new( + "data/0.lance", + vec![0, 1], + vec![0, 1], + ConcreteFileVersion::V2_0, + None, + None, + ); manifest.fragments = Arc::new(vec![fragment]); let next = apply(&manifest, vec![tombstone_field_zero()]).unwrap(); @@ -194,7 +202,7 @@ mod tests { }), Action::AddDataFile(AddDataFile { fragment: Ref::Committed(0), - file: DataFile::new_unstarted("data/fresh.lance", 1, 0), + file: DataFile::new_unstarted("data/fresh.lance", ConcreteFileVersion::V1), field_ids: vec![Ref::Local(0)], data_change: true, }), diff --git a/rust/lance-table/src/transaction/action/translate.rs b/rust/lance-table/src/transaction/action/translate.rs index 271ba6eba29..153dba7f894 100644 --- a/rust/lance-table/src/transaction/action/translate.rs +++ b/rust/lance-table/src/transaction/action/translate.rs @@ -180,6 +180,7 @@ mod tests { use crate::transaction::test_support::{ default_build_config, make_stable_row_id_manifest, sample_manifest, }; + use lance_file::version::ConcreteFileVersion; use std::sync::Arc; /// Build the same manifest twice -- once down the legacy path, once by @@ -237,9 +238,14 @@ mod tests { fn appendable_fragment(path: &str) -> Fragment { let mut fragment = Fragment::new(0); fragment.physical_rows = Some(10); - fragment - .files - .push(DataFile::new(path, vec![0], vec![0], 2, 0, None, None)); + fragment.files.push(DataFile::new( + path, + vec![0], + vec![0], + ConcreteFileVersion::V2_0, + None, + None, + )); fragment } @@ -274,9 +280,9 @@ mod tests { fn test_append_assigns_stable_row_ids_like_the_legacy_path() { let mut existing = appendable_fragment("data/1.lance"); existing.id = 1; - existing.row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&RowIdSequence::from( - 0u64..10, - )))); + existing.row_id_meta = Some(RowIdMeta::Inline( + write_row_ids(&RowIdSequence::from(0u64..10)).into(), + )); let manifest = make_stable_row_id_manifest(vec![existing]); let next = assert_parity( @@ -354,14 +360,20 @@ mod tests { "data/0b.lance", vec![1], vec![0], - 2, - 0, + ConcreteFileVersion::V2_0, None, None, )); let manifest = manifest_with_fragments(vec![fragment]); - let replacement = DataFile::new("data/0-new.lance", vec![0], vec![0], 2, 0, None, None); + let replacement = DataFile::new( + "data/0-new.lance", + vec![0], + vec![0], + ConcreteFileVersion::V2_0, + None, + None, + ); let next = assert_parity( &manifest, Operation::DataReplacement { @@ -382,7 +394,14 @@ mod tests { let operation = Operation::DataReplacement { replacements: vec![DataReplacementGroup( 0, - DataFile::new("data/0-new.lance", vec![9], vec![0], 2, 0, None, None), + DataFile::new( + "data/0-new.lance", + vec![9], + vec![0], + ConcreteFileVersion::V2_0, + None, + None, + ), )], }; let actions = Vec::::try_from(&operation).unwrap(); diff --git a/rust/lance-table/src/transaction/manifest_build.rs b/rust/lance-table/src/transaction/manifest_build.rs index 951106e2f30..5390854faae 100644 --- a/rust/lance-table/src/transaction/manifest_build.rs +++ b/rust/lance-table/src/transaction/manifest_build.rs @@ -1373,7 +1373,6 @@ impl Transaction { manifest.writer_feature_flags |= FLAG_MEM_WAL_INDEX_CATCHUP; } - match &self.operation { Operation::Overwrite { config_upsert_values: Some(tm), diff --git a/rust/lance/src/io/commit/conflict_resolver.rs b/rust/lance/src/io/commit/conflict_resolver.rs index d9eba84330b..1af930e0e04 100644 --- a/rust/lance/src/io/commit/conflict_resolver.rs +++ b/rust/lance/src/io/commit/conflict_resolver.rs @@ -4369,6 +4369,7 @@ mod tests { 1, Operation::Project { schema: dataset.schema().clone(), + preserves_nullability: false, }, );