Skip to content

feat(op-reth): proof store v2 #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
dhyaniarun1993 wants to merge 3 commits into
base: chore/proof-store-improvement
Choose a base branch
from
Draft

feat(op-reth): proof store v2 #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7bc2cdf
feat(op-reth): Proof store v2 implementation
dhyaniarun1993 Apr 4, 2026
419069e
table name refactored
dhyaniarun1993 Apr 4, 2026
e8eb221
clippy fixes
dhyaniarun1993 Apr 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions op-devstack/sysgo/mixed_runtime.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,9 @@ type MixedL2ELKind string
const DevstackL2ELKindEnvVar = "DEVSTACK_L2EL_KIND"

const (
MixedL2ELOpGeth MixedL2ELKind = "op-geth"
MixedL2ELOpReth MixedL2ELKind = "op-reth"
MixedL2ELOpGeth MixedL2ELKind = "op-geth"
MixedL2ELOpReth MixedL2ELKind = "op-reth"
MixedL2ELOpRethV2 MixedL2ELKind = "op-reth-with-proof-v2"
)

// SkipUnlessOpGeth skips the test when the L2 execution layer is op-reth
Expand Down Expand Up @@ -144,7 +145,9 @@ func NewMixedSingleChainRuntime(t devtest.T, cfg MixedSingleChainPresetConfig) *
case MixedL2ELOpGeth:
el = startL2ELNode(t, l2Net, jwtPath, jwtSecret, spec.ELKey, identity)
case MixedL2ELOpReth:
el = startMixedOpRethNode(t, l2Net, spec.ELKey, jwtPath, jwtSecret, metricsRegistrar)
el = startMixedOpRethNode(t, l2Net, spec.ELKey, jwtPath, jwtSecret, metricsRegistrar, "v1")
case MixedL2ELOpRethV2:
el = startMixedOpRethNode(t, l2Net, spec.ELKey, jwtPath, jwtSecret, metricsRegistrar, "v2")
default:
require.FailNowf("unsupported EL kind", "unsupported mixed EL kind %q", spec.ELKind)
}
Expand Down Expand Up @@ -255,6 +258,7 @@ func startMixedOpRethNode(
jwtPath string,
jwtSecret [32]byte,
metricsRegistrar L2MetricsRegistrar,
storageVersion string,
) *OpReth {
tempDir := t.TempDir()

Expand Down Expand Up @@ -330,6 +334,7 @@ func startMixedOpRethNode(
"--datadir=" + dataDirPath,
"--chain=" + chainConfigPath,
"--proofs-history.storage-path=" + proofHistoryDir,
"--proofs-history.storage-version=" + storageVersion,
}
err = exec.Command(execPath, initProofsArgs...).Run()
t.Require().NoError(err, "must init op-reth proof history")
Expand All @@ -340,6 +345,7 @@ func startMixedOpRethNode(
"--proofs-history.window=10000",
"--proofs-history.prune-interval=1m",
"--proofs-history.storage-path="+proofHistoryDir,
"--proofs-history.storage-version="+storageVersion,
)

l2EL := &OpReth{
Expand Down
2 changes: 1 addition & 1 deletion op-devstack/sysgo/singlechain_build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ func startL2ELForKey(t devtest.T, l2Net *L2Network, jwtPath string, jwtSecret [3
case MixedL2ELOpGeth:
return startL2ELNode(t, l2Net, jwtPath, jwtSecret, key, identity)
default: // op-reth
return startMixedOpRethNode(t, l2Net, key, jwtPath, jwtSecret, nil)
return startMixedOpRethNode(t, l2Net, key, jwtPath, jwtSecret, nil, "v1")
}
}

Expand Down
113 changes: 73 additions & 40 deletions rust/op-reth/crates/cli/src/commands/op_proofs/init.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@ use reth_cli_commands::common::{AccessRights, CliNodeTypes, Environment, Environ
use reth_node_core::version::version_metadata;
use reth_optimism_chainspec::OpChainSpec;
use reth_optimism_primitives::OpPrimitives;
use reth_optimism_trie::{InitializationJob, OpProofsStore, OpProofsProviderRO, db::MdbxProofsStorage};
use reth_optimism_node::args::ProofsStorageVersion;
use reth_optimism_trie::{
InitializationJob, OpProofsProviderRO, OpProofsStore,
db::{MdbxProofsStorage, MdbxProofsStorageV2},
};
use reth_provider::{BlockNumReader, DBProvider, DatabaseProviderFactory};
use std::{path::PathBuf, sync::Arc};
use tracing::info;
Expand All @@ -31,6 +35,14 @@ pub struct InitCommand<C: ChainSpecParser> {
required = true
)]
pub storage_path: PathBuf,

/// Storage schema version. Must match the version used when starting the node.
#[arg(
long = "proofs-history.storage-version",
value_name = "PROOFS_HISTORY_STORAGE_VERSION",
default_value = "v1"
)]
pub storage_version: ProofsStorageVersion,
}

impl<C: ChainSpecParser<ChainSpec = OpChainSpec>> InitCommand<C> {
Expand All @@ -44,51 +56,72 @@ impl<C: ChainSpecParser<ChainSpec = OpChainSpec>> InitCommand<C> {
// Initialize the environment with read-only access
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RO)?;

// Create the proofs storage without the metrics wrapper.
// During initialization we write billions of entries; the metrics layer's
// `AtomicBucket::push` (used by `Histogram::record_many`) is append-only and
// would accumulate ~19 bytes per observation, causing OOM on large chains.
let storage: Arc<MdbxProofsStorage> = Arc::new(
MdbxProofsStorage::new(&self.storage_path)
.map_err(|e| eyre::eyre!("Failed to create MdbxProofsStorage: {e}"))?,
);

// Check if already initialized
if let Some((block_number, block_hash)) = storage.provider_ro()?.get_earliest_block_number()? {
info!(
target: "reth::cli",
block_number = block_number,
block_hash = ?block_hash,
"Proofs storage already initialized"
);
return Ok(());
}
macro_rules! run_init {
($storage:expr) => {{
let storage = $storage;

// Get the current chain state
let ChainInfo { best_number, best_hash, .. } = provider_factory.chain_info()?;
// Check if already initialized
if let Some((block_number, block_hash)) =
storage.provider_ro()?.get_earliest_block_number()?
{
info!(
target: "reth::cli",
block_number = block_number,
block_hash = ?block_hash,
"Proofs storage already initialized"
);
return Ok(());
}

info!(
target: "reth::cli",
best_number = best_number,
best_hash = ?best_hash,
"Starting backfill job for current chain state"
);
// Get the current chain state
let ChainInfo { best_number, best_hash, .. } = provider_factory.chain_info()?;

// Run the backfill job
{
let db_provider =
provider_factory.database_provider_ro()?.disable_long_read_transaction_safety();
let db_tx = db_provider.into_tx();
info!(
target: "reth::cli",
best_number = best_number,
best_hash = ?best_hash,
"Starting backfill job for current chain state"
);

InitializationJob::new(storage, db_tx).run(best_number, best_hash)?;
// Run the backfill job
{
let db_provider = provider_factory
.database_provider_ro()?
.disable_long_read_transaction_safety();
let db_tx = db_provider.into_tx();

InitializationJob::new(storage, db_tx).run(best_number, best_hash)?;
}

info!(
target: "reth::cli",
best_number = best_number,
best_hash = ?best_hash,
"Proofs storage initialized successfully"
);
}};
}

info!(
target: "reth::cli",
best_number = best_number,
best_hash = ?best_hash,
"Proofs storage initialized successfully"
);
// Create the proofs storage without the metrics wrapper.
// During initialization we write billions of entries; the metrics layer's
// `AtomicBucket::push` (used by `Histogram::record_many`) is append-only and
// would accumulate ~19 bytes per observation, causing OOM on large chains.
match self.storage_version {
ProofsStorageVersion::V1 => {
let storage: Arc<MdbxProofsStorage> = Arc::new(
MdbxProofsStorage::new(&self.storage_path)
.map_err(|e| eyre::eyre!("Failed to create MdbxProofsStorage: {e}"))?,
);
run_init!(storage);
}
ProofsStorageVersion::V2 => {
let storage: Arc<MdbxProofsStorageV2> = Arc::new(
MdbxProofsStorageV2::new(&self.storage_path)
.map_err(|e| eyre::eyre!("Failed to create MdbxProofsStorageV2: {e}"))?,
);
run_init!(storage);
}
}

Ok(())
}
Expand Down
74 changes: 53 additions & 21 deletions rust/op-reth/crates/cli/src/commands/op_proofs/prune.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ use reth_cli_commands::common::{AccessRights, CliNodeTypes, Environment, Environ
use reth_node_core::version::version_metadata;
use reth_optimism_chainspec::OpChainSpec;
use reth_optimism_primitives::OpPrimitives;
use reth_optimism_trie::{OpProofStoragePruner, OpProofsStore, OpProofsProviderRO, db::MdbxProofsStorage};
use reth_optimism_node::args::ProofsStorageVersion;
use reth_optimism_trie::{
OpProofStoragePruner, OpProofsProviderRO, OpProofsStore,
db::{MdbxProofsStorage, MdbxProofsStorageV2},
};
use std::{path::PathBuf, sync::Arc};
use tracing::info;

Expand Down Expand Up @@ -41,6 +45,14 @@ pub struct PruneCommand<C: ChainSpecParser> {
value_name = "PROOFS_HISTORY_PRUNE_BATCH_SIZE"
)]
pub proofs_history_prune_batch_size: u64,

/// Storage schema version. Must match the version used when starting the node.
#[arg(
long = "proofs-history.storage-version",
value_name = "PROOFS_HISTORY_STORAGE_VERSION",
default_value = "v1"
)]
pub storage_version: ProofsStorageVersion,
}

impl<C: ChainSpecParser<ChainSpec = OpChainSpec>> PruneCommand<C> {
Expand All @@ -54,28 +66,48 @@ impl<C: ChainSpecParser<ChainSpec = OpChainSpec>> PruneCommand<C> {
// Initialize the environment with read-only access
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RO)?;

let storage: Arc<MdbxProofsStorage> = Arc::new(
MdbxProofsStorage::new(&self.storage_path)
.map_err(|e| eyre::eyre!("Failed to create MdbxProofsStorage: {e}"))?,
);
macro_rules! run_prune {
($storage:expr) => {{
let storage = $storage;

let provider_ro = storage.provider_ro()?;
let earliest_block = provider_ro.get_earliest_block_number()?;
let latest_block = provider_ro.get_latest_block_number()?;
info!(
target: "reth::cli",
?earliest_block,
?latest_block,
"Current proofs storage block range"
);
drop(provider_ro);

let pruner = OpProofStoragePruner::new(
storage,
provider_factory,
self.proofs_history_window,
self.proofs_history_prune_batch_size,
);
pruner.run();
}};
}

let provider_ro = storage.provider_ro()?;
let earliest_block = provider_ro.get_earliest_block_number()?;
let latest_block = provider_ro.get_latest_block_number()?;
info!(
target: "reth::cli",
?earliest_block,
?latest_block,
"Current proofs storage block range"
);
match self.storage_version {
ProofsStorageVersion::V1 => {
let storage: Arc<MdbxProofsStorage> = Arc::new(
MdbxProofsStorage::new(&self.storage_path)
.map_err(|e| eyre::eyre!("Failed to create MdbxProofsStorage: {e}"))?,
);
run_prune!(storage);
}
ProofsStorageVersion::V2 => {
let storage: Arc<MdbxProofsStorageV2> = Arc::new(
MdbxProofsStorageV2::new(&self.storage_path)
.map_err(|e| eyre::eyre!("Failed to create MdbxProofsStorageV2: {e}"))?,
);
run_prune!(storage);
}
}

let pruner = OpProofStoragePruner::new(
storage,
provider_factory,
self.proofs_history_window,
self.proofs_history_prune_batch_size,
);
pruner.run();
Ok(())
}
}
Expand Down
71 changes: 50 additions & 21 deletions rust/op-reth/crates/cli/src/commands/op_proofs/unwind.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ use reth_cli_commands::common::{AccessRights, CliNodeTypes, Environment, Environ
use reth_node_core::version::version_metadata;
use reth_optimism_chainspec::OpChainSpec;
use reth_optimism_primitives::OpPrimitives;
use reth_optimism_trie::{OpProofsStore, OpProofsProviderRO, OpProofsProviderRw, db::MdbxProofsStorage};
use reth_optimism_node::args::ProofsStorageVersion;
use reth_optimism_trie::{
OpProofsProviderRO, OpProofsProviderRw, OpProofsStore,
db::{MdbxProofsStorage, MdbxProofsStorageV2},
};
use reth_provider::{BlockReader, TransactionVariant};
use std::{path::PathBuf, sync::Arc};
use tracing::{info, warn};
Expand All @@ -32,6 +36,14 @@ pub struct UnwindCommand<C: ChainSpecParser> {
/// All history *after* this block will be removed.
#[arg(long, value_name = "TARGET_BLOCK")]
pub target: u64,

/// Storage schema version. Must match the version used when starting the node.
#[arg(
long = "proofs-history.storage-version",
value_name = "PROOFS_HISTORY_STORAGE_VERSION",
default_value = "v1"
)]
pub storage_version: ProofsStorageVersion,
}

impl<C: ChainSpecParser> UnwindCommand<C> {
Expand Down Expand Up @@ -70,28 +82,45 @@ impl<C: ChainSpecParser<ChainSpec = OpChainSpec>> UnwindCommand<C> {
// Initialize the environment with read-only access
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RO)?;

// Create the proofs storage
let storage: Arc<MdbxProofsStorage> = Arc::new(
MdbxProofsStorage::new(&self.storage_path)
.map_err(|e| eyre::eyre!("Failed to create MdbxProofsStorage: {e}"))?,
);

// Validate that the target block is within a valid range for unwinding
if !self.validate_unwind_range(storage.clone())? {
return Ok(());
macro_rules! run_unwind {
($storage:expr) => {{
let storage = $storage;

// Validate that the target block is within a valid range for unwinding
if !self.validate_unwind_range(storage.clone())? {
return Ok(());
}

// Get the target block from the main database
let block = provider_factory
.recovered_block(self.target.into(), TransactionVariant::NoHash)?
.ok_or_else(|| {
eyre::eyre!("Target block {} not found in the main database", self.target)
})?;

info!(target: "reth::cli", block_number = block.number, block_hash = %block.hash(), "Unwinding to target block");
let provider_rw = storage.provider_rw()?;
provider_rw.unwind_history(block.block_with_parent())?;
provider_rw.commit()?;
}};
}

// Get the target block from the main database
let block = provider_factory
.recovered_block(self.target.into(), TransactionVariant::NoHash)?
.ok_or_else(|| {
eyre::eyre!("Target block {} not found in the main database", self.target)
})?;

info!(target: "reth::cli", block_number = block.number, block_hash = %block.hash(), "Unwinding to target block");
let provider_rw = storage.provider_rw()?;
provider_rw.unwind_history(block.block_with_parent())?;
provider_rw.commit()?;
match self.storage_version {
ProofsStorageVersion::V1 => {
let storage: Arc<MdbxProofsStorage> = Arc::new(
MdbxProofsStorage::new(&self.storage_path)
.map_err(|e| eyre::eyre!("Failed to create MdbxProofsStorage: {e}"))?,
);
run_unwind!(storage);
}
ProofsStorageVersion::V2 => {
let storage: Arc<MdbxProofsStorageV2> = Arc::new(
MdbxProofsStorageV2::new(&self.storage_path)
.map_err(|e| eyre::eyre!("Failed to create MdbxProofsStorageV2: {e}"))?,
);
run_unwind!(storage);
}
}

Ok(())
}
Expand Down
Loading