Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
10654ef
feat(transaction): add the Transaction V2 action vocabulary
wjones127 Aug 15, 2026
b00cf9f
feat(transaction): carry action-based operations through the wire
wjones127 Aug 16, 2026
466286d
refactor(table): extract manifest assembly helpers from build_manifest
wjones127 Aug 16, 2026
5641887
feat(transaction): apply the minting actions
wjones127 Aug 16, 2026
a44c884
feat(transaction): apply the reference-stable actions
wjones127 Aug 16, 2026
3313acd
feat(transaction): translate Append, Delete, and UpdateBases into act…
wjones127 Aug 16, 2026
40524b8
feat(transaction): translate DataReplacement into actions
wjones127 Aug 16, 2026
f013217
feat(transaction): compute conflict footprints for action sets
wjones127 Aug 16, 2026
9edfb63
feat(commit): resolve action-set conflicts by footprint
wjones127 Aug 16, 2026
6d11202
test(transaction): cover relocating an action set onto a newer version
wjones127 Aug 16, 2026
b7c7d1b
test(commit): end-to-end composite transactions against a real dataset
wjones127 Aug 16, 2026
993e53f
feat(transaction): add the DropField action
wjones127 Aug 17, 2026
dc0ae2b
refactor(transaction): give each action its own module
wjones127 Aug 17, 2026
cbefa2c
feat(transaction): add the ReserveFragmentIds action
wjones127 Aug 17, 2026
5b66944
feat(transaction): add the ResetTable action
wjones127 Aug 17, 2026
d4eb9e1
feat(transaction): add the ConfigUpdate action
wjones127 Aug 17, 2026
9e814fe
refactor(transaction): rename UserOperation to CompositeOperation
wjones127 Aug 18, 2026
aa751d7
docs(transaction): document when action version and data_change field…
wjones127 Aug 18, 2026
f079ca2
refactor(transaction): generate the action dispatch from one list
wjones127 Aug 18, 2026
badd4f5
test(lance): move composite transaction tests into the library
wjones127 Aug 18, 2026
64cb96d
docs(transaction): give the real reason SetDeletionFile takes no Ref
wjones127 Aug 18, 2026
75bef6d
refactor(transaction): split applying actions from assembling the man…
wjones127 Aug 18, 2026
2aaa1ec
feat(transaction): reference fields by Ref in the field actions
wjones127 Aug 18, 2026
25f0610
fix(transaction): adapt the action path to upstream API changes
wjones127 Aug 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 14 additions & 16 deletions protos/transaction/actions.proto
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ package lance.table;
/*
* Action-based transactions (Transaction V2) — DRAFT.
*
* A `UserOperation` replaces the single legacy `Operation` (see
* A `CompositeOperation` replaces the single legacy `Operation` (see
* transaction.proto) with an ordered list of granular `Action`s that commit
* atomically as one manifest change. This file is the wire draft only.
*
* STATUS
* ------
* The messages and field numbers here are NOT yet a stable contract. Library
* support is read-side fail-closed only: a transaction carrying a
* `UserOperation` is rejected on load, and there is no write path. Apply,
* `CompositeOperation` is rejected on load, and there is no write path. Apply,
* id translation, and conflict resolution are intentionally absent.
*
* DESIGN RATIONALE
Expand Down Expand Up @@ -106,10 +106,10 @@ package lance.table;
* id) that may not be committed yet.
*
* `committed` is an already-assigned id. `local` is a placeholder token minted
* by an `Add*` action earlier in the same `UserOperation`; it resolves to a
* by an `Add*` action earlier in the same `CompositeOperation`; it resolves to a
* freshly-allocated committed id at apply, and re-resolves against the target's
* counters on merge/rebase. `local` tokens are scoped to a single
* `UserOperation` and must be distinct within it (validated by the writer once
* `CompositeOperation` and must be distinct within it (validated by the writer once
* the write path exists).
*/
message Ref {
Expand All @@ -123,23 +123,21 @@ message Ref {
* A user-facing, composable transaction: an ordered list of user actions that
* commit atomically as a single manifest change.
*/
message UserOperation {
// Human-readable description, e.g. "INSERT INTO t VALUES (1)".
string description = 1;
message CompositeOperation {
// Unique identifier for this operation (matches Transaction.uuid semantics).
string uuid = 2;
string uuid = 1;
// The dataset version this operation was planned against.
uint64 read_version = 3;
uint64 read_version = 2;
// The ordered list of user actions applied by this operation.
repeated UserAction actions = 4;
repeated UserAction actions = 3;
}

/*
* A single user-recognizable step within a UserOperation (e.g. "append batch",
* A single user-recognizable step within a CompositeOperation (e.g. "append batch",
* "rebuild index").
*
* The description keeps the transaction history human-readable. When a range of
* transactions is squashed, each original UserOperation collapses into one
* transactions is squashed, each original CompositeOperation collapses into one
* UserAction so the readable sequence survives; the action lists are flattened
* when applied to the manifest.
*/
Expand Down Expand Up @@ -286,8 +284,8 @@ message AddBase {
*/
message TombstoneFieldData {
Ref fragment = 1;
// Committed field ids whose current backing is tombstoned.
repeated uint64 field_ids = 2;
// The fields whose current backing is tombstoned.
repeated Ref field_ids = 2;
// Data-change marker; see AddFragment.data_change.
optional bool data_change = 3;
}
Expand Down Expand Up @@ -376,7 +374,7 @@ message UpdateCompactedSsTables {
* no live fields). References an existing committed field id.
*/
message DropField {
uint64 field = 1;
Ref field = 1;
}

/*
Expand All @@ -389,7 +387,7 @@ message DropField {
* AddDataFile to rewrite the data. New facets are added as optional fields.
*/
message AlterField {
uint64 field = 1;
Ref field = 1;
optional string name = 2;
// New Arrow logical type (see Field.logical_type). The cast.
optional string logical_type = 3;
Expand Down
2 changes: 1 addition & 1 deletion protos/transaction/transaction.proto
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ message Transaction {
DataOverlay data_overlay = 115;
// Action-based transaction (Transaction V2). See actions.proto.
// DRAFT: currently rejected on load; no write path.
UserOperation user_operation = 116;
CompositeOperation composite_operation = 116;
}

// Fields 200/202 (`blob_append` / `blob_overwrite`) previously represented blob dataset ops.
Expand Down
30 changes: 17 additions & 13 deletions rust/lance-table/src/format/fragment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,22 @@ impl TryFrom<pb::DeletionFile> for DeletionFile {
}
}

impl From<&DeletionFile> for pb::DeletionFile {
fn from(value: &DeletionFile) -> Self {
let file_type = match value.file_type {
DeletionFileType::Array => pb::deletion_file::DeletionFileType::ArrowArray,
DeletionFileType::Bitmap => pb::deletion_file::DeletionFileType::Bitmap,
};
Self {
read_version: value.read_version,
id: value.id,
file_type: file_type.into(),
num_deleted_rows: value.num_deleted_rows.unwrap_or_default() as u64,
base_id: value.base_id,
}
}
}

/// Data fragment.
///
/// A fragment is a set of files which represent the different columns of the same rows.
Expand Down Expand Up @@ -716,19 +732,7 @@ impl TryFrom<pb::DataFragment> for Fragment {

impl From<&Fragment> for pb::DataFragment {
fn from(f: &Fragment) -> Self {
let deletion_file = f.deletion_file.as_ref().map(|f| {
let file_type = match f.file_type {
DeletionFileType::Array => pb::deletion_file::DeletionFileType::ArrowArray,
DeletionFileType::Bitmap => pb::deletion_file::DeletionFileType::Bitmap,
};
pb::DeletionFile {
read_version: f.read_version,
id: f.id,
file_type: file_type.into(),
num_deleted_rows: f.num_deleted_rows.unwrap_or_default() as u64,
base_id: f.base_id,
}
});
let deletion_file = f.deletion_file.as_ref().map(pb::DeletionFile::from);

let row_id_sequence = f.row_id_meta.as_ref().map(|m| match m {
RowIdMeta::Inline(data) => {
Expand Down
2 changes: 2 additions & 0 deletions rust/lance-table/src/transaction.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
//! ```text
//! builder Transaction: an operation plus the version it was based on
//! operation the vocabulary of changes an operation can describe
//! action the finer-grained Transaction V2 vocabulary (draft)
//! update_map incremental edits to the manifest's string maps
//! validate pre-commit checks against the manifest being replaced
//! manifest_build applying an operation to produce the next manifest
Expand All @@ -26,6 +27,7 @@
//! proto the persisted protobuf encoding of all of the above
//! ```

pub mod action;
mod builder;
mod conflicts;
mod index_maintenance;
Expand Down
Loading
Loading