diff --git a/README.md b/README.md index 5e911aa16..03cc016ca 100644 --- a/README.md +++ b/README.md @@ -65,11 +65,19 @@ In addition to electrs's original configuration options, a few new options are a - `--http-addr ` - HTTP server address/port to listen on (default: `127.0.0.1:3000`). - `--lightmode` - enable light mode (see above) - `--cors ` - origins allowed to make cross-site request (optional, defaults to none). -- `--address-search` - enables the by-prefix address search index. +- `--address-search` - enables the by-prefix address search index. Best-effort: entries are + written when a block is indexed but are not removed if that block is later reorged out, so + the index may return addresses with no transaction in the current best chain. Run a full + reindex to clear stale entries if this matters for your deployment. - `--index-unspendables` - enables indexing of provably unspendable outputs. - `--enable-mining-rest` - enables cached mining-related HTTP endpoints. - `--utxos-limit ` - maximum number of utxos to return per address. +- `--history-scan-limit ` - approximate number of history rows scanned per address lookup before failing with a "too popular" error; progress is checkpointed so repeated lookups make headway. Must be at least `1`. Default: `100000`. +- `--utxos-checkpoint-limit ` - maximum number of utxos held in a partially scanned utxo set and its saved checkpoint, and of unconfirmed history entries scanned per address. Must be at least `--utxos-limit`. Default: 10 times `--utxos-limit`. - `--electrum-txs-limit ` - maximum number of txs to return per address in the electrum server (does not apply for the http api). +- `--electrum-rpc-write-timeout ` - maximum time to transmit a complete Electrum reply, including a whole batch and its newline, an error reply, or a subscription notification. The deadline starts at the first write and is not extended by partial progress. Expiry closes the connection and releases its response buffers. Idle subscriptions and command execution are unaffected. Default: `30`, matching the RPC proxy's `CLIENT_WRITE_TIMEOUT_SECS`; `0` disables the bound, which is only accepted with `--electrum-rpc-conn-max-age` set or the global response budget disabled. The `electrum_client_write_timeouts_total` counter records connections closed by this deadline. +- `--electrum-rpc-max-response-num-bytes ` - maximum size, in bytes, of a single Electrum solicited reply line (brackets, commas, error objects, and newline included). Overflowing replies return a correlated error (code 1) and the rest of the batch is not executed; the command that overflowed may already have completed. Subscription notifications are exempt. `0` disables the cap. Default: `8388608` (8 MiB) on Bitcoin, `33554432` (32 MiB) on Liquid (sized for a 2016-header window during a dynafed parameter vote). +- `--electrum-rpc-global-response-budget-bytes ` - aggregate cap, in bytes, on Electrum solicited-reply buffer memory retained across all connections at any instant, including batch replies held while later batch elements execute. Subscription notifications and queued request lines are not counted. Small replies below a 16 KiB per-connection floor are exempt; above the floor each reply charges the budget in 64 KiB chunks (or a smaller final chunk) and releases on drop. Requests that would exceed the budget are rejected with a server error (code 2) instead of allocating, and the connection stays usable. Must be `>= --electrum-rpc-max-response-num-bytes`, and requires a non-zero `--electrum-rpc-write-timeout` or `--electrum-rpc-conn-max-age`. `0` disables the budget. Default: `67108864` (64 MiB) on Bitcoin, `268435456` (256 MiB) on Liquid. - `--electrum-banner ` - welcome banner text for electrum server. Additional options with the `liquid` feature: diff --git a/src/bin/electrs.rs b/src/bin/electrs.rs index 6f94d028b..b051a7b1b 100644 --- a/src/bin/electrs.rs +++ b/src/bin/electrs.rs @@ -8,7 +8,7 @@ extern crate log; extern crate electrs; use crossbeam_channel::{self as channel}; -use error_chain::ChainedError; +use error_chain::{bail, ChainedError}; use std::{env, process, thread}; use std::sync::{Arc, RwLock}; use std::time::Duration; @@ -20,7 +20,9 @@ use electrs::{ electrum::RPC as ElectrumRPC, errors::*, metrics::Metrics, - new_index::{precache, zmq, ChainQuery, FetchFrom, Indexer, Mempool, Query, Store}, + new_index::{ + precache, zmq, ChainQuery, FetchFrom, Indexer, Mempool, MempoolSyncStatus, Query, Store, + }, rest, signal::Waiter, }; @@ -35,6 +37,14 @@ use electrs::metrics::MetricOpts; /// Default salt rotation interval in seconds (24 hours) const DEFAULT_SALT_ROTATION_INTERVAL_SECS: u64 = 24 * 3600; +/// Maximum number of `FailedToIndex` retries during the initial mempool +/// sync before giving up. +const MAX_INITIAL_MEMPOOL_SYNC_RETRIES: u32 = 100; + +/// Maximum number of consecutive `FailedToIndex` main loop cycles before giving up, +/// about an hour at the 5 second loop interval. +const MAX_CONSECUTIVE_MEMPOOL_SYNC_FAILURES: u32 = 720; + fn fetch_from(config: &Config, store: &Store) -> FetchFrom { let mut jsonrpc_import = config.jsonrpc_import; if !jsonrpc_import { @@ -122,18 +132,32 @@ fn run_server(config: Arc, salt_rwlock: Arc>) -> Result<( Arc::clone(&config), ))); - // A mempool update is retried whenever the tip moves, so index and try again. An update - // that cannot advance the tip never will here, and no listener is bound yet. - while !Mempool::update(&mempool, &daemon, &tip)? { - let new_tip = indexer.update(&daemon)?; - if new_tip == tip { - warn!( - "index could not advance, starting up with a partial index tip='{}'", - tip - ); - break; + let mut mempool_sync_retries = 0; + loop { + match Mempool::update(&mempool, &daemon, &tip)? { + MempoolSyncStatus::Synced => break, + MempoolSyncStatus::TipMoved => { + let new_tip = indexer.update(&daemon)?; + if new_tip == tip { + warn!( + "index could not advance, starting up with a partial index tip='{}'", + tip + ); + break; + } + tip = new_tip; + } + MempoolSyncStatus::FailedToIndex => { + mempool_sync_retries += 1; + if mempool_sync_retries >= MAX_INITIAL_MEMPOOL_SYNC_RETRIES { + bail!( + "initial mempool sync failed to index {} times, giving up", + MAX_INITIAL_MEMPOOL_SYNC_RETRIES + ); + } + signal.wait(Duration::from_secs(3), false)?; + } } - tip = new_tip; } #[cfg(feature = "liquid")] @@ -168,6 +192,7 @@ fn run_server(config: Arc, salt_rwlock: Arc>) -> Result<( "count of iterations of electrs main loop each 5 seconds or after interrupts", )); + let mut mempool_sync_failures = 0; loop { main_loop_count.inc(); @@ -185,8 +210,24 @@ fn run_server(config: Arc, salt_rwlock: Arc>) -> Result<( }; // Update mempool - if !Mempool::update(&mempool, &daemon, &tip)? { - warn!("skipped failed mempool update, trying again in 5 seconds"); + match Mempool::update(&mempool, &daemon, &tip)? { + MempoolSyncStatus::Synced => mempool_sync_failures = 0, + MempoolSyncStatus::TipMoved => { + warn!("mempool sync aborted: chain tip moved, retrying next cycle"); + } + MempoolSyncStatus::FailedToIndex => { + mempool_sync_failures += 1; + if mempool_sync_failures >= MAX_CONSECUTIVE_MEMPOOL_SYNC_FAILURES { + bail!( + "mempool sync failed to index {} consecutive times, giving up", + MAX_CONSECUTIVE_MEMPOOL_SYNC_FAILURES + ); + } + warn!( + "mempool_sync_failed_to_index: retrying next cycle failures='{}'", + mempool_sync_failures + ); + } } // Update subscribed clients diff --git a/src/chain.rs b/src/chain.rs index 0475223e6..680b45f57 100644 --- a/src/chain.rs +++ b/src/chain.rs @@ -2,7 +2,7 @@ pub use bitcoin::{ address, blockdata::block::Header as BlockHeader, blockdata::script, consensus::deserialize, hash_types::TxMerkleNode, Address, Block, BlockHash, OutPoint, ScriptBuf as Script, Sequence, - Transaction, TxIn, TxOut, Txid, + Transaction, TxIn, TxOut, Txid, Wtxid, }; #[cfg(feature = "liquid")] @@ -11,6 +11,7 @@ pub use { elements::{ address, confidential, encode::deserialize, script, Address, AssetId, Block, BlockHash, BlockHeader, OutPoint, Script, Sequence, Transaction, TxIn, TxMerkleNode, TxOut, Txid, + Wtxid, }, }; diff --git a/src/config.rs b/src/config.rs index 409d1c09c..91491aee8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -60,7 +60,10 @@ pub struct Config { pub cookie: Option, pub electrum_rpc_addr: SocketAddr, pub electrum_rpc_conn_max_age: Option, + pub electrum_rpc_write_timeout: Option, pub electrum_rpc_max_request_num_bytes: usize, + pub electrum_rpc_max_response_num_bytes: usize, + pub electrum_rpc_global_response_budget_bytes: usize, pub http_addr: SocketAddr, pub http_socket_file: Option, pub monitoring_addr: SocketAddr, @@ -72,9 +75,12 @@ pub struct Config { pub cors: Option, pub precache_scripts: Option, pub utxos_limit: usize, + pub history_scan_limit: usize, + pub utxos_checkpoint_limit: usize, pub electrum_txs_limit: usize, pub electrum_subscription_limit: usize, pub electrum_checkpoint_proof_concurrency_limit: usize, + pub electrum_checkpoint_merkle_cache_mb: usize, pub electrum_banner: String, pub rpc_logging: RpcLogging, pub zmq_addr: Option, @@ -135,6 +141,24 @@ fn str_to_socketaddr(address: &str, what: &str) -> SocketAddr { .unwrap() } +fn value_or_default_or_exit( + m: &clap::ArgMatches, + arg_name: &str, + cli_name: &str, + default: usize, +) -> usize { + match m.value_of(arg_name) { + None => default, + Some(raw) => raw.parse::().unwrap_or_else(|_| { + clap::Error::with_description( + &format!("--{} expects a non-negative integer (got {:?})", cli_name, raw), + clap::ErrorKind::ValueValidation, + ) + .exit() + }), + } +} + impl Config { pub fn from_args() -> Config { let network_help = format!("Select network type ({})", Network::names().join(", ")); @@ -195,6 +219,13 @@ impl Config { .default_value("0") .takes_value(true), ) + .arg( + Arg::with_name("electrum_rpc_write_timeout") + .long("electrum-rpc-write-timeout") + .help("Maximum time (in seconds) to transmit a complete Electrum response, including a whole batch, errors, or a notification. Starts at the first write and is not extended by partial progress. Closes stalled connections and releases response buffers. Does not time idle subscriptions or command execution. 0 = disabled, which requires --electrum-rpc-conn-max-age or an unlimited --electrum-rpc-global-response-budget-bytes (default: 30).") + .default_value("30") + .takes_value(true), + ) .arg( Arg::with_name("electrum_rpc_max_request_num_bytes") .long("electrum-rpc-max-request-num-bytes") @@ -202,6 +233,18 @@ impl Config { .default_value("1048576") .takes_value(true), ) + .arg( + Arg::with_name("electrum_rpc_max_response_num_bytes") + .long("electrum-rpc-max-response-num-bytes") + .help("Maximum size (in bytes) of a single Electrum RPC solicited reply line, including brackets, commas, error objects, and terminating newline. Applies to singles, batches, parse errors, and invalid-request errors; subscription notifications are exempt. When a reply cannot fit, the client receives a correlated error (code 1) for the overflowing element and later batch elements are not executed. 0 = unlimited (default: 8388608, i.e. 8 MiB on Bitcoin; 33554432, i.e. 32 MiB on Liquid, sized for a 2016-header window during a dynafed parameter vote).") + .takes_value(true), + ) + .arg( + Arg::with_name("electrum_rpc_global_response_budget_bytes") + .long("electrum-rpc-global-response-budget-bytes") + .help("Aggregate cap (in bytes) on Electrum solicited-reply buffer memory retained across all connections at any instant, including batch replies held while later batch elements execute and replies held while writer threads block on non-reading clients. Subscription notifications and queued request lines are not counted. Small replies (< 16 KiB per connection) are exempt; above that floor each reply charges the budget in 64 KiB chunks (or a smaller final chunk) and releases on drop. Requests that would exceed the budget are rejected with a server error (code 2) instead of allocating. Must be >= --electrum-rpc-max-response-num-bytes so a single maximum-size reply can succeed, and requires a non-zero --electrum-rpc-write-timeout or --electrum-rpc-conn-max-age so stalled clients release their share. 0 = unlimited (default: 67108864, i.e. 64 MiB on Bitcoin; 268435456, i.e. 256 MiB on Liquid, keeping the same ratio to the per-line cap).") + .takes_value(true), + ) .arg( Arg::with_name("http_addr") .long("http-addr") @@ -282,6 +325,18 @@ impl Config { .help("Maximum number of utxos to process per address. Lookups for addresses with more utxos will fail. Applies to the Electrum and HTTP APIs.") .default_value("500") ) + .arg( + Arg::with_name("history_scan_limit") + .long("history-scan-limit") + .help("Approximate number of scripthash history rows scanned per utxo/stats lookup before giving up with a \"too popular\" error (the scan reads through the end of the height it is on when the limit is reached, so it may process somewhat more). Unlike --utxos-limit and --electrum-txs-limit, which cap the size of the result, this bounds the amount of work done per request regardless of the outcome. Must be at least 1.") + .default_value("100000") + ) + .arg( + Arg::with_name("utxos_checkpoint_limit") + .long("utxos-checkpoint-limit") + .help("Maximum number of utxos held in a partially scanned utxo set, and in the checkpoint saved for it. An address whose utxo set grows past this while scanning fails with a \"too popular\" error and no checkpoint is saved. Also bounds the unconfirmed history entries scanned per address for utxo and stats lookups. Must be at least --utxos-limit (default: 10 times --utxos-limit).") + .takes_value(true) + ) .arg( Arg::with_name("electrum_txs_limit") .long("electrum-txs-limit") @@ -296,8 +351,13 @@ impl Config { ).arg( Arg::with_name("electrum_checkpoint_proof_concurrency_limit") .long("electrum-checkpoint-proof-concurrency-limit") - .help("Maximum number of blockchain.block.header(s) checkpoint Merkle proof builds (triggered by a non-zero cp_height) allowed to run at once, process-wide. Each build hashes every header from genesis up to cp_height, so an unbounded count lets concurrent cheap requests pin every CPU core. Requests past the cap fail immediately rather than queueing. 0 = reject all such requests.") - .default_value("2") + .help("Maximum number of blockchain.block.header(s) checkpoint Merkle proof tree rebuilds (a non-zero cp_height not already cached) allowed to run at once, process-wide. Rebuilds past the cap fail immediately rather than queueing. Cached lookups (the common case once a cp_height has been served once) are never subject to this limit. Default: half the detected CPU cores (min 1). 0 = reject all rebuilds.") + .takes_value(true) + ).arg( + Arg::with_name("electrum_checkpoint_merkle_cache_mb") + .long("electrum-checkpoint-merkle-cache-mb") + .help("Approximate memory budget, in MB, for checkpoint Merkle proof handling: cached trees and the working memory of concurrent rebuilds share this one budget (independently of electrum-checkpoint-proof-concurrency-limit, so a high core count can no longer multiply memory use past it). Cached trees use at most half of it, so a full cache still leaves room for rebuilds. It is a soft target based on an estimate, not a hard ceiling. Entry size scales with cp_height (up to ~28MB near the chain tip). A cp_height too large to ever fit is served without being cached.") + .default_value("256") .takes_value(true) ).arg( Arg::with_name("electrum_banner") @@ -450,6 +510,12 @@ impl Config { #[cfg(feature = "liquid")] Network::LiquidRegtest => 51401, }; + #[cfg(not(feature = "liquid"))] + let default_electrum_rpc_max_response_num_bytes: usize = 8_388_608; + #[cfg(feature = "liquid")] + let default_electrum_rpc_max_response_num_bytes: usize = 33_554_432; + let default_electrum_rpc_global_response_budget_bytes: usize = + default_electrum_rpc_max_response_num_bytes.saturating_mul(8); let default_http_port = match network_type { #[cfg(not(feature = "liquid"))] Network::Bitcoin => 3000, @@ -514,6 +580,10 @@ impl Config { 0 => None, // 0 = unlimited / never disconnect secs => Some(Duration::from_secs(secs)), }; + let electrum_rpc_write_timeout = match value_t_or_exit!(m, "electrum_rpc_write_timeout", u64) { + 0 => None, + secs => Some(Duration::from_secs(secs)), + }; let electrum_rpc_max_request_num_bytes: usize = match value_t_or_exit!( m, "electrum_rpc_max_request_num_bytes", @@ -522,6 +592,75 @@ impl Config { 0 => usize::MAX, // 0 = unlimited bytes => bytes, }; + let electrum_rpc_max_response_num_bytes: usize = match value_or_default_or_exit( + &m, + "electrum_rpc_max_response_num_bytes", + "electrum-rpc-max-response-num-bytes", + default_electrum_rpc_max_response_num_bytes, + ) { + 0 => usize::MAX, // 0 = unlimited + bytes => bytes, + }; + let electrum_rpc_global_response_budget_bytes: usize = match value_or_default_or_exit( + &m, + "electrum_rpc_global_response_budget_bytes", + "electrum-rpc-global-response-budget-bytes", + default_electrum_rpc_global_response_budget_bytes, + ) { + 0 => usize::MAX, // 0 = unlimited + bytes => bytes, + }; + if electrum_rpc_global_response_budget_bytes != usize::MAX + && electrum_rpc_max_response_num_bytes != usize::MAX + && electrum_rpc_global_response_budget_bytes < electrum_rpc_max_response_num_bytes + { + clap::Error::with_description( + &format!( + "--electrum-rpc-global-response-budget-bytes ({}) must be >= \ + --electrum-rpc-max-response-num-bytes ({}); a single maximum-size \ + reply cannot succeed otherwise", + electrum_rpc_global_response_budget_bytes, + electrum_rpc_max_response_num_bytes, + ), + clap::ErrorKind::ValueValidation, + ) + .exit(); + } + if electrum_rpc_global_response_budget_bytes != usize::MAX + && electrum_rpc_write_timeout.is_none() + && electrum_rpc_conn_max_age.is_none() + { + clap::Error::with_description( + "--electrum-rpc-global-response-budget-bytes requires a non-zero \ + --electrum-rpc-write-timeout or --electrum-rpc-conn-max-age; otherwise \ + clients that stop reading hold their share of the budget until restart", + clap::ErrorKind::ValueValidation, + ) + .exit(); + } + let utxos_limit = value_t_or_exit!(m, "utxos_limit", usize); + let history_scan_limit = value_t_or_exit!(m, "history_scan_limit", usize); + if history_scan_limit == 0 { + clap::Error::with_description( + "--history-scan-limit must be at least 1", + clap::ErrorKind::ValueValidation, + ) + .exit(); + } + let utxos_checkpoint_limit = match m.value_of("utxos_checkpoint_limit") { + Some(_) => value_t_or_exit!(m, "utxos_checkpoint_limit", usize), + None => utxos_limit.saturating_mul(10), + }; + if utxos_checkpoint_limit < utxos_limit { + clap::Error::with_description( + &format!( + "--utxos-checkpoint-limit ({}) must be >= --utxos-limit ({})", + utxos_checkpoint_limit, utxos_limit + ), + clap::ErrorKind::ValueValidation, + ) + .exit(); + } let http_addr: SocketAddr = str_to_socketaddr( m.value_of("http_addr") .unwrap_or(&format!("127.0.0.1:{}", default_http_port)), @@ -590,15 +729,32 @@ impl Config { daemon_parallelism: value_t_or_exit!(m, "daemon_parallelism", usize), daemon_conn_max_age, cookie, - utxos_limit: value_t_or_exit!(m, "utxos_limit", usize), + utxos_limit, + history_scan_limit, + utxos_checkpoint_limit, electrum_rpc_addr, electrum_rpc_conn_max_age, + electrum_rpc_write_timeout, electrum_rpc_max_request_num_bytes, + electrum_rpc_max_response_num_bytes, + electrum_rpc_global_response_budget_bytes, electrum_txs_limit: value_t_or_exit!(m, "electrum_txs_limit", usize), electrum_subscription_limit: value_t_or_exit!(m, "electrum_subscription_limit", usize), - electrum_checkpoint_proof_concurrency_limit: value_t_or_exit!( + electrum_checkpoint_proof_concurrency_limit: match m + .value_of("electrum_checkpoint_proof_concurrency_limit") + { + Some(v) => v.parse::().unwrap_or_else(|_| { + clap::Error::value_validation_auto(format!( + "The argument '{}' isn't a valid value", + v + )) + .exit() + }), + None => (num_cpus::get() / 2).max(1), + }, + electrum_checkpoint_merkle_cache_mb: value_t_or_exit!( m, - "electrum_checkpoint_proof_concurrency_limit", + "electrum_checkpoint_merkle_cache_mb", usize ), electrum_banner, diff --git a/src/daemon.rs b/src/daemon.rs index 7e4442e73..ece3e937b 100644 --- a/src/daemon.rs +++ b/src/daemon.rs @@ -23,7 +23,7 @@ use elements::encode::{deserialize, serialize_hex}; use electrs_macros::trace; -use crate::chain::{Block, BlockHash, BlockHeader, Network, Transaction, Txid}; +use crate::chain::{Block, BlockHash, BlockHeader, Network, Transaction, Txid, Wtxid}; use crate::metrics::{CounterVec, HistogramOpts, HistogramVec, MetricOpts, Metrics}; use crate::signal::Waiter; use crate::util::{HeaderList, DEFAULT_BLOCKHASH}; @@ -87,8 +87,25 @@ lazy_static! { const MAX_ATTEMPTS: u32 = 5; const RETRY_WAIT_DURATION: Duration = Duration::from_secs(1); +const RECONNECT_RETRY_WAIT_DURATION: Duration = Duration::from_secs(3); const BLOCK_TEMPLATE_RPC_TIMEOUT: Duration = Duration::from_secs(30); +fn retry_reconnect_with_backoff(mut reconnect: F, mut wait: W) -> Result +where + F: FnMut() -> Result, + W: FnMut(Duration) -> Result<()>, +{ + loop { + match reconnect() { + Ok(value) => return Ok(value), + Err(e) => { + warn!("failed connecting to RPC daemon: {}", e.display_chain()); + wait(RECONNECT_RETRY_WAIT_DURATION)?; + } + } + } +} + #[trace] fn parse_hash(value: &Value) -> Result where @@ -228,6 +245,19 @@ impl SubmitPackageResult { .collect() } + /// (txid, wtxid) of the accepted transactions that are in the daemon's mempool exactly as + /// submitted. `other-wtxid` results are excluded: a same-txid transaction with a different + /// witness was already there and the submitted one was ignored. + pub fn accepted_as_submitted(&self) -> Vec<(Txid, Wtxid)> { + self.tx_results + .iter() + .filter(|(_, tx)| tx.error.is_none() && tx.other_wtxid.is_none()) + .filter_map(|(wtxid, tx)| { + Some((Txid::from_str(&tx.txid).ok()?, Wtxid::from_str(wtxid).ok()?)) + }) + .collect() + } + /// Build the response for the Electrum `blockchain.transaction.broadcast_package` method. /// /// When `verbose` is true, the full `submitpackage` result is returned. Otherwise a compact @@ -1181,16 +1211,11 @@ impl Daemon { } #[trace] - fn retry_reconnect(&self) -> Daemon { - // XXX add a max reconnection attempts limit? - loop { - match self.reconnect() { - Ok(daemon) => break daemon, - Err(e) => { - warn!("failed connecting to RPC daemon: {}", e.display_chain()); - } - } - } + fn retry_reconnect(&self) -> Result { + retry_reconnect_with_backoff( + || self.reconnect(), + |duration| self.signal.wait(duration, false), + ) } // Send requests in parallel over multiple RPC connections as individual JSON-RPC requests (with no JSON-RPC batching), @@ -1219,9 +1244,15 @@ impl Daemon { // get initialized as necessary for the `rpc_threads` pool thread managed by rayon. thread_local!(static DAEMON_INSTANCE: OnceCell = OnceCell::new()); - DAEMON_INSTANCE.with(|daemon| { + DAEMON_INSTANCE.with(|daemon| -> Result { + if daemon.get().is_none() { + daemon + .set(self.retry_reconnect()?) + .unwrap_or_else(|_| unreachable!("thread-local cell was just empty")); + } daemon - .get_or_init(|| self.retry_reconnect()) + .get() + .expect("just initialized above") .retry_request(&method, ¶ms) }) }) @@ -1385,9 +1416,11 @@ impl Daemon { self.request("getrawtransaction", json!([txid, verbose, blockhash])) } + /// Fetch a mempool transaction on behalf of an API client broadcast/package path. #[trace] pub fn getmempooltx(&self, txhash: &Txid) -> Result { - let value = self.request("getrawtransaction", json!([txhash, /*verbose=*/ false]))?; + let value = + self.request_proxied("getrawtransaction", json!([txhash, /*verbose=*/ false]))?; tx_from_value(value) } @@ -1597,8 +1630,9 @@ impl Daemon { #[cfg(test)] mod tests { use super::{ - parse_jsonrpc_reply, recycle_due, BlockingSemaphore, Connection, ConnectionConfig, - CookieGetter, DAEMON_MAX_BODY_BYTES, DAEMON_MAX_HEADER_LINE_BYTES, + parse_jsonrpc_reply, recycle_due, retry_reconnect_with_backoff, BlockingSemaphore, + Connection, ConnectionConfig, CookieGetter, DAEMON_MAX_BODY_BYTES, + DAEMON_MAX_HEADER_LINE_BYTES, }; use crate::errors::{Error, ErrorKind, Result}; use crate::signal::Waiter; @@ -1620,6 +1654,79 @@ mod tests { Duration::from_millis(n) } + #[test] + fn accepted_as_submitted_skips_errors_and_other_witnesses() { + use super::SubmitPackageResult; + use crate::chain::{Txid, Wtxid}; + use std::str::FromStr; + + let id = |n: u8| format!("{:02x}", n).repeat(32); + let result: SubmitPackageResult = serde_json::from_value(json!({ + "package_msg": "success", + "tx-results": { + id(1): { "txid": id(11), "vsize": 100, "fees": { "base": 0.0001 } }, + id(2): { "txid": id(12), "other-wtxid": id(3) }, + id(4): { "txid": id(14), "error": "bad-txns-inputs-missingorspent" }, + }, + })) + .unwrap(); + + assert_eq!( + result.accepted_as_submitted(), + vec![( + Txid::from_str(&id(11)).unwrap(), + Wtxid::from_str(&id(1)).unwrap() + )] + ); + assert_eq!(result.accepted_txids().len(), 2); + } + + #[test] + fn reconnect_failures_are_backed_off_before_retrying() { + let mut attempts = 0; + let mut waits = vec![]; + + let result = retry_reconnect_with_backoff( + || { + attempts += 1; + if attempts < 3 { + bail!("post-connect setup failed"); + } + Ok(42) + }, + |duration| { + waits.push(duration); + Ok(()) + }, + ) + .unwrap(); + + assert_eq!(result, 42); + assert_eq!(attempts, 3); + assert_eq!(waits, vec![secs(3), secs(3)]); + } + + #[test] + fn reconnect_retry_stops_when_backoff_is_interrupted() { + let mut attempts = 0; + + let error = retry_reconnect_with_backoff::<(), _, _>( + || { + attempts += 1; + assert_eq!(attempts, 1, "reconnect retried after interrupted backoff"); + bail!("post-connect setup failed") + }, + |duration| { + assert_eq!(duration, secs(3)); + bail!("shutdown requested") + }, + ) + .unwrap_err(); + + assert_eq!(attempts, 1); + assert!(error.to_string().contains("shutdown requested")); + } + /// Spawn a fake daemon that runs `respond` against the accepted socket, and hand back a /// one-shot `Connection` pointed at it. The connection's I/O timeout doubles as its /// whole-request `recv` budget, so `io_timeout` is what the deadline tests turn on. diff --git a/src/electrum/client_io.rs b/src/electrum/client_io.rs new file mode 100644 index 000000000..b62ce4278 --- /dev/null +++ b/src/electrum/client_io.rs @@ -0,0 +1,387 @@ +//! One deadline for all writes making up a client-facing response. + +use std::io::{self, Write}; +use std::net::{Shutdown, TcpStream}; +use std::time::{Duration, Instant}; + +use crate::errors::*; +use crate::metrics::Counter; + +pub(super) fn write_to_client( + stream: &TcpStream, + timeout: Option, + timeouts: &Counter, + write: impl FnOnce(&mut ClientWriter<'_>) -> Result<()>, +) -> Result<()> { + if timeout.is_none() { + stream + .set_write_timeout(None) + .chain_err(|| "failed to disable client write timeout")?; + } + let mut writer = ClientWriter { + stream, + timeout, + deadline: None, + timed_out: false, + }; + let result = write(&mut writer); + if writer.timed_out { + timeouts.inc(); + // Preserve the classification even when serde wrapped the I/O error + // while streaming a budget rejection. + Err(ErrorKind::ClientWriteTimeout.into()) + } else { + result + } +} + +pub(super) struct ClientWriter<'a> { + stream: &'a TcpStream, + timeout: Option, + // Start at the first write, after command execution and reply assembly. + deadline: Option, + timed_out: bool, +} + +impl ClientWriter<'_> { + fn expired(&mut self) -> io::Error { + if !self.timed_out { + self.timed_out = true; + // Discard queued output on close instead of leaving a FIN behind + // bytes the client will not read. Only timed-out writes abort this way. + if let Err(error) = socket2::SockRef::from(self.stream).set_linger(Some(Duration::ZERO)) + { + warn!( + "failed to enable abortive close after client write timeout: {}", + error + ); + } + // Wake the reader; teardown closes all socket handles to send the RST. + let _ = self.stream.shutdown(Shutdown::Both); + } + io::Error::new(io::ErrorKind::TimedOut, "client response write timed out") + } +} + +impl Write for ClientWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.timed_out { + return Err(self.expired()); + } + if bytes.is_empty() { + return Ok(0); + } + let timeout = match self.timeout { + Some(timeout) => timeout, + None => return (&*self.stream).write(bytes), + }; + let now = Instant::now(); + let deadline = match self.deadline { + Some(deadline) => deadline, + None => { + let deadline = now.checked_add(timeout).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "client write timeout is too large", + ) + })?; + self.deadline = Some(deadline); + deadline + } + }; + if now >= deadline { + return Err(self.expired()); + } + + // A fresh full timeout on each partial write would let a trickling + // reader extend the response indefinitely. Only use the remaining time. + self.stream.set_write_timeout(Some(deadline - now))?; + match (&*self.stream).write(bytes) { + Err(error) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut + ) => + { + Err(self.expired()) + } + result => result, + } + } + + fn flush(&mut self) -> io::Result<()> { + if self.timed_out { + return Err(self.expired()); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read; + use std::net::TcpListener; + use std::sync::{mpsc, Arc}; + use std::thread; + + // A watchdog bounds the test even if the write deadline regresses. + struct SocketPair { + server: Arc, + client: TcpStream, + stop: mpsc::Sender<()>, + watchdog: Option>, + } + + impl SocketPair { + fn new() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (server, _) = listener.accept().unwrap(); + let server = Arc::new(server); + socket2::SockRef::from(&*server) + .set_send_buffer_size(4096) + .unwrap(); + client + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let (stop, receiver) = mpsc::channel(); + let socket = Arc::clone(&server); + let watchdog = thread::spawn(move || { + if receiver.recv_timeout(Duration::from_secs(5)).is_err() { + let _ = socket.shutdown(Shutdown::Both); + } + }); + Self { + server, + client, + stop, + watchdog: Some(watchdog), + } + } + } + + impl Drop for SocketPair { + fn drop(&mut self) { + let _ = self.server.shutdown(Shutdown::Both); + let _ = self.client.shutdown(Shutdown::Both); + let _ = self.stop.send(()); + self.watchdog.take().unwrap().join().unwrap(); + } + } + + fn counter() -> Counter { + Counter::new("client_write_timeouts", "Test timeouts").unwrap() + } + + #[test] + fn stalled_writer_times_out_without_the_client_disconnecting() { + let sockets = SocketPair::new(); + socket2::SockRef::from(&sockets.client) + .set_recv_buffer_size(4096) + .unwrap(); + let timeouts = counter(); + let start = Instant::now(); + let result = write_to_client( + &sockets.server, + Some(Duration::from_millis(150)), + &timeouts, + |writer| { + writer + .write_all(&vec![b'x'; 2 * 1024 * 1024]) + .chain_err(|| "write failed") + }, + ); + assert!(matches!( + result.unwrap_err().kind(), + ErrorKind::ClientWriteTimeout + )); + assert!(start.elapsed() < Duration::from_secs(3)); + assert_eq!(timeouts.get(), 1); + // sockets.client is still open and has never read anything. + } + + #[test] + fn reading_client_receives_payload_and_newline() { + let mut sockets = SocketPair::new(); + let timeouts = counter(); + write_to_client( + &sockets.server, + Some(Duration::from_secs(1)), + &timeouts, + |writer| { + writer.write_all(b"hello").chain_err(|| "payload failed")?; + writer.write_all(b"\n").chain_err(|| "newline failed") + }, + ) + .unwrap(); + let mut bytes = [0; 6]; + sockets.client.read_exact(&mut bytes).unwrap(); + assert_eq!(&bytes, b"hello\n"); + assert_eq!(timeouts.get(), 0); + } + + #[test] + fn delayed_reader_can_finish_and_disabled_timeout_clears_the_socket_limit() { + for timeout in [Some(Duration::from_secs(4)), None] { + let sockets = SocketPair::new(); + sockets + .server + .set_write_timeout(Some(Duration::from_millis(10))) + .unwrap(); + let mut client = sockets.client.try_clone().unwrap(); + let reader = thread::spawn(move || { + thread::sleep(Duration::from_millis(150)); + let mut bytes = Vec::new(); + client.read_to_end(&mut bytes).unwrap(); + bytes + }); + let timeouts = counter(); + let payload = vec![b'x'; 512 * 1024]; + let result = write_to_client(&sockets.server, timeout, &timeouts, |writer| { + writer.write_all(&payload).chain_err(|| "write failed") + }); + let _ = sockets.server.shutdown(Shutdown::Write); + let received = reader.join().unwrap(); + result.unwrap(); + assert_eq!(received, payload); + assert_eq!(timeouts.get(), 0); + } + } + + #[test] + fn chunks_and_newline_share_one_deadline() { + let sockets = SocketPair::new(); + let timeouts = counter(); + let result = write_to_client( + &sockets.server, + Some(Duration::from_secs(1)), + &timeouts, + |writer| { + assert!( + writer.deadline.is_none(), + "assembly must not start the write clock" + ); + writer + .write_all(b"payload") + .chain_err(|| "payload failed")?; + assert!(writer.deadline.is_some()); + // Advance the deadline explicitly instead of racing a short sleep. + writer.deadline = Some(Instant::now() - Duration::from_secs(1)); + let newline = writer.write_all(b"\n").chain_err(|| "newline failed"); + assert!(writer.write_all(b"retry").is_err()); + newline + }, + ); + assert!(matches!( + result.unwrap_err().kind(), + ErrorKind::ClientWriteTimeout + )); + assert_eq!(timeouts.get(), 1); + } + + #[test] + fn write_accepted_by_the_kernel_is_not_aborted_at_the_deadline() { + let sockets = SocketPair::new(); + let timeouts = counter(); + let result = write_to_client( + &sockets.server, + Some(Duration::from_millis(50)), + &timeouts, + |writer| { + writer + .write_all(b"payload") + .chain_err(|| "payload failed")?; + thread::sleep(Duration::from_millis(100)); + writer.flush().chain_err(|| "flush failed") + }, + ); + assert!(result.is_ok(), "completed write was treated as a timeout"); + assert_eq!(timeouts.get(), 0); + let mut client = sockets.client.try_clone().unwrap(); + let mut bytes = [0; 7]; + client.read_exact(&mut bytes).unwrap(); + assert_eq!(&bytes, b"payload"); + } + + #[test] + fn partial_progress_does_not_extend_the_deadline() { + let sockets = SocketPair::new(); + socket2::SockRef::from(&sockets.client) + .set_recv_buffer_size(4096) + .unwrap(); + let mut client = sockets.client.try_clone().unwrap(); + let (stop, receiver) = mpsc::channel(); + let reader = thread::spawn(move || { + let mut total = 0; + let mut bytes = [0; 1024]; + loop { + match client.read(&mut bytes) { + Ok(0) | Err(_) => break, + Ok(n) => total += n, + } + if receiver.recv_timeout(Duration::from_millis(10)).is_ok() { + break; + } + } + total + }); + let timeouts = counter(); + let start = Instant::now(); + let result = write_to_client( + &sockets.server, + Some(Duration::from_millis(250)), + &timeouts, + |writer| { + writer + .write_all(&vec![b'x'; 2 * 1024 * 1024]) + .chain_err(|| "write failed") + }, + ); + let _ = stop.send(()); + assert!(reader.join().unwrap() > 0); + assert!(matches!( + result.unwrap_err().kind(), + ErrorKind::ClientWriteTimeout + )); + assert!(start.elapsed() < Duration::from_secs(3)); + assert_eq!(timeouts.get(), 1); + } + + #[test] + fn streaming_budget_errors_and_notifications_use_the_same_write_deadline() { + use crate::electrum::response::{ResponseBudget, ResponseWriter}; + + for notification in [false, true] { + let sockets = SocketPair::new(); + socket2::SockRef::from(&sockets.client) + .set_recv_buffer_size(4096) + .unwrap(); + let timeouts = counter(); + let mut responses = ResponseWriter::new(2 * 1024 * 1024, ResponseBudget::new(1)); + let result = write_to_client( + &sockets.server, + Some(Duration::from_millis(150)), + &timeouts, + |writer| { + if notification { + responses + .send_notification(writer, &json!({"params":["x".repeat(1024 * 1024)]})) + } else { + responses.send( + writer, + Ok(json!({"id":"x".repeat(1024 * 1024), "method":"server.ping"})), + |_| panic!("budget rejection must not execute the command"), + |_| {}, + ) + } + }, + ); + assert!(matches!( + result.unwrap_err().kind(), + ErrorKind::ClientWriteTimeout + )); + assert_eq!(timeouts.get(), 1); + } + } +} diff --git a/src/electrum/mod.rs b/src/electrum/mod.rs index d28426d7d..499b9c241 100644 --- a/src/electrum/mod.rs +++ b/src/electrum/mod.rs @@ -1,3 +1,5 @@ +mod client_io; +mod response; mod server; pub use server::RPC; diff --git a/src/electrum/response.rs b/src/electrum/response.rs new file mode 100644 index 000000000..a1847d213 --- /dev/null +++ b/src/electrum/response.rs @@ -0,0 +1,934 @@ +use std::io::{self, BufWriter, Write}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use serde::Serialize; +use serde_json::Value; + +use crate::errors::*; + +const MAX_ARRAY_BATCH: usize = 20; +const BUFFER_FLOOR: usize = 16 * 1024; +const BUDGET_CHUNK: usize = 64 * 1024; +const PER_LINE_MSG: &str = "response line exceeds max_response_bytes"; +const GLOBAL_MSG: &str = "response budget exhausted, retry later"; +const SKIPPED_MSG: &str = "batch response budget exhausted; not executed"; + +const _: () = { + assert!(SKIPPED_MSG.len() >= PER_LINE_MSG.len()); + assert!(SKIPPED_MSG.len() >= GLOBAL_MSG.len()); +}; + +// JSON-RPC errors plus Electrum application error codes. +#[repr(i16)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum JsonRpcV2Error { + ParseError = -32700, + InvalidRequest = -32600, + MethodNotFound = -32601, + InvalidParams = -32602, + InternalError = -32603, + BadRequest = 1, + DaemonError = 2, +} + +impl JsonRpcV2Error { + pub(super) fn into_i16(self) -> i16 { + self as i16 + } +} + +#[derive(Serialize)] +struct ErrorEnvelope<'a> { + error: ErrorBody<'a>, + id: &'a Value, + jsonrpc: &'static str, +} + +#[derive(Serialize)] +struct ErrorBody<'a> { + code: i16, + message: &'a str, +} + +impl<'a> ErrorEnvelope<'a> { + fn new(message: &'a str, id: Option<&'a Value>, code: JsonRpcV2Error) -> Self { + Self { + error: ErrorBody { + code: code.into_i16(), + message, + }, + id: id.unwrap_or(&Value::Null), + jsonrpc: "2.0", + } + } + + fn encoded_len(&self) -> usize { + let mut counter = ByteCounter(0); + serde_json::to_writer(&mut counter, self).expect("counting JSON values cannot fail"); + counter.0 + } +} + +pub(super) fn json_rpc_error( + message: impl std::fmt::Display, + id: Option<&Value>, + code: JsonRpcV2Error, +) -> Value { + serde_json::to_value(ErrorEnvelope::new(&message.to_string(), id, code)) + .expect("an error envelope contains only JSON values") +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ResponseLimit { + PerLine, + Global, +} + +impl ResponseLimit { + fn message(self) -> &'static str { + match self { + Self::PerLine => PER_LINE_MSG, + Self::Global => GLOBAL_MSG, + } + } + + fn code(self) -> JsonRpcV2Error { + match self { + Self::PerLine => JsonRpcV2Error::BadRequest, + Self::Global => JsonRpcV2Error::DaemonError, + } + } + + fn error(self) -> Error { + match self { + Self::PerLine => ErrorKind::PerLineResponseOverflow.into(), + Self::Global => ErrorKind::ResponseBudgetExhausted.into(), + } + } +} + +pub(super) struct ResponseBudget { + limit: usize, + reserved: AtomicUsize, +} + +impl ResponseBudget { + pub(super) fn new(limit: usize) -> Arc { + Arc::new(Self { + limit, + reserved: AtomicUsize::new(0), + }) + } +} + +struct Reservation { + budget: Arc, + granted: usize, +} + +impl Reservation { + fn ensure(&mut self, total: usize) -> std::result::Result<(), ResponseLimit> { + if self.budget.limit == usize::MAX || total <= self.granted { + return Ok(()); + } + let required = total - self.granted; + let preferred = required + .checked_next_multiple_of(BUDGET_CHUNK) + .unwrap_or(required); + let mut reserved = self.budget.reserved.load(Ordering::Relaxed); + loop { + let available = self.budget.limit - reserved; + if required > available { + return Err(ResponseLimit::Global); + } + let grant = preferred.min(available); + match self.budget.reserved.compare_exchange_weak( + reserved, + reserved + grant, + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => { + self.granted += grant; + return Ok(()); + } + Err(latest) => reserved = latest, + } + } + } +} + +impl Drop for Reservation { + fn drop(&mut self) { + self.budget + .reserved + .fetch_sub(self.granted, Ordering::AcqRel); + } +} + +// Fields drop in declaration order: free the buffer before releasing its grant. +struct ResponseBuffer { + bytes: Vec, + reservation: Reservation, + limit: usize, +} + +impl ResponseBuffer { + fn new(bytes: Vec, budget: Arc, limit: usize) -> Self { + Self { + bytes, + reservation: Reservation { budget, granted: 0 }, + limit, + } + } + + fn reserve(&mut self, line_bytes: usize) -> std::result::Result<(), ResponseLimit> { + self.reservation + .ensure(line_bytes.saturating_sub(BUFFER_FLOOR)) + } + + fn encode( + &mut self, + value: &impl Serialize, + tail: usize, + ) -> std::result::Result<(), EncodeError> { + let limit = self + .limit + .checked_sub(tail) + .ok_or(EncodeError::Limited(ResponseLimit::PerLine))?; + let start = self.bytes.len(); + let mut writer = LimitedWriter { + buffer: self, + limit, + tail, + failure: None, + }; + match serde_json::to_writer(&mut writer, value) { + Ok(()) => Ok(()), + Err(error) => { + writer.buffer.bytes.truncate(start); + Err(match writer.failure { + Some(limit) => EncodeError::Limited(limit), + None => EncodeError::Json(error), + }) + } + } + } + + fn framing(&mut self, bytes: &[u8]) -> Result<()> { + let limit = self.limit; + let mut writer = LimitedWriter { + buffer: self, + limit, + tail: 0, + failure: None, + }; + writer + .write_all(bytes) + .map_err(|_| writer.failure.expect("bounded write failed").error()) + } + + fn recycle(mut self) -> Vec { + cleanup_buffer(&mut self.bytes); + std::mem::take(&mut self.bytes) + } +} + +enum EncodeError { + Limited(ResponseLimit), + Json(serde_json::Error), +} + +impl From for Error { + fn from(error: EncodeError) -> Self { + match error { + EncodeError::Limited(limit) => limit.error(), + EncodeError::Json(error) => Error::with_chain(error, "failed to serialize response"), + } + } +} + +struct LimitedWriter<'a> { + buffer: &'a mut ResponseBuffer, + limit: usize, + tail: usize, + failure: Option, +} + +impl Write for LimitedWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if bytes.is_empty() { + return Ok(0); + } + let result = self + .buffer + .bytes + .len() + .checked_add(bytes.len()) + .filter(|len| *len <= self.limit) + .ok_or(ResponseLimit::PerLine) + .and_then(|len| { + self.buffer.reserve(len.saturating_add(self.tail))?; + Ok(len) + }); + let len = match result { + Ok(len) => len, + Err(limit) => { + self.failure = Some(limit); + return Err(io::Error::new(io::ErrorKind::Other, limit.message())); + } + }; + if len > self.buffer.bytes.capacity() { + // Avoid Vec's implicit doubling: every byte above the exempt floor + // must be charged, including capacity retained after a rollback. + let capacity = if len <= BUFFER_FLOOR { + BUFFER_FLOOR + } else if self.buffer.reservation.budget.limit == usize::MAX { + len.checked_next_multiple_of(BUDGET_CHUNK).unwrap_or(len) + } else { + self.buffer.reservation.granted.saturating_add(BUFFER_FLOOR) + } + .min(self.limit); + self.buffer + .bytes + .reserve_exact(capacity - self.buffer.bytes.len()); + } + self.buffer.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +struct ByteCounter(usize); + +impl Write for ByteCounter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0 = self.0.saturating_add(bytes.len()); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +enum Delivery { + Buffered, + BudgetErrors { ids: Vec, batch: bool }, +} + +impl Delivery { + fn budget_errors(requests: Vec, batch: bool) -> Self { + let ids = requests + .into_iter() + .map(|mut request| { + request + .as_object_mut() + .and_then(|object| object.remove("id")) + .unwrap_or(Value::Null) + }) + .collect(); + Self::BudgetErrors { ids, batch } + } + + fn send(self, stream: &mut impl Write, buffer: &ResponseBuffer) -> Result<()> { + match self { + Self::Buffered => stream + .write_all(&buffer.bytes) + .chain_err(|| format!("failed to write {} response bytes", buffer.bytes.len())), + Self::BudgetErrors { ids, batch } => { + let mut writer = BufWriter::with_capacity(BUFFER_FLOOR, stream); + if batch { + writer + .write_all(b"[") + .chain_err(|| "failed to send batch framing")?; + } + for (i, id) in ids.iter().enumerate() { + if i > 0 { + writer + .write_all(b",") + .chain_err(|| "failed to send batch framing")?; + } + let error = + ErrorEnvelope::new(GLOBAL_MSG, Some(id), JsonRpcV2Error::DaemonError); + serde_json::to_writer(&mut writer, &error) + .chain_err(|| "failed to send budget error")?; + } + writer + .write_all(if batch { b"]\n" } else { b"\n" }) + .chain_err(|| "failed to send response framing")?; + writer.flush().chain_err(|| "failed to send budget errors") + } + } + } +} + +pub(super) struct ResponseWriter { + buffer: Vec, + budget: Arc, + limit: usize, +} + +impl ResponseWriter { + pub(super) fn new(limit: usize, budget: Arc) -> Self { + Self { + buffer: Vec::new(), + budget, + limit, + } + } + + pub(super) fn send( + &mut self, + stream: &mut impl Write, + request: serde_json::Result, + mut execute: impl FnMut(&Value) -> Value, + mut rejected: impl FnMut(ResponseLimit), + ) -> Result<()> { + let mut buffer = ResponseBuffer::new( + std::mem::take(&mut self.buffer), + Arc::clone(&self.budget), + self.limit, + ); + let assembled = match request { + Ok(Value::Array(commands)) => { + assemble_batch(commands, &mut buffer, &mut execute, &mut rejected) + } + Ok(command) => assemble_single(command, &mut buffer, &mut execute, &mut rejected), + Err(_) => { + let error = ErrorEnvelope::new("parse error", None, JsonRpcV2Error::ParseError); + buffer + .encode(&error, 1) + .map_err(|error| { + if let EncodeError::Limited(limit) = &error { + rejected(*limit); + } + Error::from(error) + }) + .and_then(|()| buffer.framing(b"\n")) + .map(|()| Delivery::Buffered) + } + }; + let result = assembled.and_then(|delivery| delivery.send(stream, &buffer)); + self.buffer = buffer.recycle(); + result + } + + // Notifications are exempt from byte limits; the caller still bounds writes. + pub(super) fn send_notification( + &mut self, + stream: &mut impl Write, + value: &Value, + ) -> Result<()> { + let result = serde_json::to_writer(&mut self.buffer, value) + .chain_err(|| "failed to serialize notification") + .and_then(|()| { + self.buffer.push(b'\n'); + stream + .write_all(&self.buffer) + .chain_err(|| "failed to send notification") + }); + cleanup_buffer(&mut self.buffer); + result + } +} + +fn try_reserve_single_error_response( + buffer: &mut ResponseBuffer, + id: Option<&Value>, + rejected: &mut impl FnMut(ResponseLimit), +) -> Result { + let error_size = ErrorEnvelope::new(PER_LINE_MSG, id, JsonRpcV2Error::BadRequest) + .encoded_len() + .checked_add(1) + .ok_or_else(|| ResponseLimit::PerLine.error())?; + if let Err(limit) = buffer.reserve(error_size) { + rejected(limit); + if error_size > buffer.limit { + return Err(ResponseLimit::PerLine.error()); + } + buffer.bytes = Vec::new(); + return Ok(false); + } + Ok(true) +} + +fn assemble_single( + command: Value, + buffer: &mut ResponseBuffer, + execute: &mut impl FnMut(&Value) -> Value, + rejected: &mut impl FnMut(ResponseLimit), +) -> Result { + if buffer.limit == 0 { + return Err(ResponseLimit::PerLine.error()); + } + let id = command.get("id"); + if !try_reserve_single_error_response(buffer, id, rejected)? { + return Ok(Delivery::budget_errors(vec![command], false)); + } + match buffer.encode(&execute(&command), 1) { + Ok(()) => {} + Err(EncodeError::Limited(limit)) => { + rejected(limit); + buffer.encode(&ErrorEnvelope::new(limit.message(), id, limit.code()), 1)?; + } + Err(error) => return Err(error.into()), + } + buffer.framing(b"\n")?; + Ok(Delivery::Buffered) +} + +fn try_reserve_batch_error_responses( + commands: &[Value], + buffer: &mut ResponseBuffer, + rejected: &mut impl FnMut(ResponseLimit), +) -> Result> { + let n = commands.len(); + let mut error_sizes = [0; MAX_ARRAY_BATCH]; + let mut later_errors = 0usize; + for (command, size) in commands.iter().zip(error_sizes.iter_mut()) { + *size = ErrorEnvelope::new(SKIPPED_MSG, command.get("id"), JsonRpcV2Error::BadRequest) + .encoded_len(); + later_errors = later_errors + .checked_add(*size) + .ok_or("batch reserve overflow")?; + } + // '[' + (n - 1) commas + ']\n'. + let all_errors_size = later_errors + .checked_add(n + 2) + .ok_or("batch framing overflow")?; + if all_errors_size > buffer.limit { + bail!( + "cannot fit correlated error envelopes for batch of {} \ + under electrum_rpc_max_response_num_bytes={}", + n, + buffer.limit + ); + } + if let Err(limit) = buffer.reserve(all_errors_size) { + rejected(limit); + buffer.bytes = Vec::new(); + return Ok(None); + } + + Ok(Some(error_sizes)) +} + +fn assemble_batch( + commands: Vec, + buffer: &mut ResponseBuffer, + execute: &mut impl FnMut(&Value) -> Value, + rejected: &mut impl FnMut(ResponseLimit), +) -> Result { + let n = commands.len(); + if n > MAX_ARRAY_BATCH { + bail!( + "Too many elements in batch requests {} max:{}", + n, + MAX_ARRAY_BATCH + ); + } + if n == 0 { + buffer.framing(b"[]\n")?; + return Ok(Delivery::Buffered); + } + + let Some(error_sizes) = try_reserve_batch_error_responses(&commands, buffer, rejected)? else { + return Ok(Delivery::budget_errors(commands, true)); + }; + let mut later_errors = error_sizes.iter().sum::(); + + buffer.framing(b"[")?; + let mut overflow: Option = None; + for (i, command) in commands.into_iter().enumerate() { + if i > 0 { + buffer.framing(b",")?; + } + later_errors -= error_sizes[i]; + let tail = later_errors + (n - 1 - i) + b"]\n".len(); + let (message, code) = match overflow { + Some(limit) => (SKIPPED_MSG, limit.code()), + None => match buffer.encode(&execute(&command), tail) { + Ok(()) => continue, + Err(EncodeError::Limited(limit)) => { + rejected(limit); + overflow = Some(limit); + (limit.message(), limit.code()) + } + Err(error) => return Err(error.into()), + }, + }; + buffer.encode(&ErrorEnvelope::new(message, command.get("id"), code), tail)?; + } + buffer.framing(b"]\n")?; + Ok(Delivery::Buffered) +} + +fn cleanup_buffer(buffer: &mut Vec) { + buffer.clear(); + if buffer.capacity() > BUFFER_FLOOR { + *buffer = Vec::new(); + } +} + +pub(super) fn warn_limits(max_request_bytes: usize, max_response_bytes: usize, txs_limit: usize) { + if max_response_bytes == usize::MAX { + return; + } + let worst_reserve = max_request_bytes + .saturating_add(MAX_ARRAY_BATCH * 128) + .saturating_add(MAX_ARRAY_BATCH + 2); + if worst_reserve >= max_response_bytes { + warn!( + "electrum_rpc_max_response_num_bytes={} is not large enough to guarantee \ + correlated error replies for a full batch of requests up to \ + electrum_rpc_max_request_num_bytes={}; such batches will be rejected by \ + closing the connection", + max_response_bytes, max_request_bytes + ); + } + const HISTORY_ENTRY_ESTIMATE_BYTES: usize = 128; + if txs_limit.saturating_mul(HISTORY_ENTRY_ESTIMATE_BYTES) >= max_response_bytes { + warn!( + "electrum_txs_limit={} can generate histories exceeding \ + electrum_rpc_max_response_num_bytes={} (rough estimate: {} bytes per \ + entry, IDs/UTXOs/transactions/headers excluded); legitimate history \ + replies can be rejected under this configuration", + txs_limit, max_response_bytes, HISTORY_ENTRY_ESTIMATE_BYTES + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Barrier; + + fn send_reply( + limit: usize, + budget: usize, + reply: &Value, + ) -> (Value, usize, Vec) { + let mut writer = ResponseWriter::new(limit, ResponseBudget::new(budget)); + let mut bytes = Vec::new(); + let mut rejected = Vec::new(); + writer + .send( + &mut bytes, + Ok(json!({"id": reply["id"]})), + |_| reply.clone(), + |limit| rejected.push(limit), + ) + .unwrap(); + assert!(bytes.ends_with(b"\n")); + assert_eq!(writer.budget.reserved.load(Ordering::Relaxed), 0); + assert!(writer.buffer.capacity() <= BUFFER_FLOOR); + ( + serde_json::from_slice(&bytes).unwrap(), + bytes.len(), + rejected, + ) + } + + #[test] + fn single_limit_includes_escaped_id_and_newline() { + let reply = json!({"id": "a\n\"\\é", "jsonrpc": "2.0", "result": "x".repeat(256)}); + let exact = serde_json::to_vec(&reply).unwrap().len() + 1; + let (actual, size, rejected) = send_reply(exact, usize::MAX, &reply); + assert_eq!(actual, reply); + assert_eq!(size, exact); + assert!(rejected.is_empty()); + + let (actual, size, rejected) = send_reply(exact - 1, usize::MAX, &reply); + assert_eq!(actual["id"], reply["id"]); + assert_eq!(actual["error"]["code"], 1); + assert!(size < exact); + assert_eq!(rejected, [ResponseLimit::PerLine]); + } + + #[test] + fn partial_final_grants_allow_replies_below_both_limits() { + for limit in [32_768, 100_000] { + let reply = json!({"id": 1, "result": "x".repeat(limit - 100)}); + let (actual, size, rejected) = send_reply(limit, limit, &reply); + assert_eq!(actual, reply); + assert!(size <= limit); + assert!(rejected.is_empty()); + } + } + + #[test] + fn framing_and_rollback_keep_buffer_capacity_charged() { + for suffix in [b"\n".as_slice(), b"]\n".as_slice(), b",".as_slice()] { + let budget = ResponseBudget::new(100_000); + let mut buffer = ResponseBuffer::new(Vec::new(), Arc::clone(&budget), 100_000); + buffer + .encode(&json!("x".repeat(90_000)), suffix.len()) + .map_err(Error::from) + .unwrap(); + assert!(buffer + .encode(&json!("y".repeat(20_000)), suffix.len()) + .is_err()); + buffer.framing(suffix).unwrap(); + assert!(buffer.bytes.capacity() <= buffer.reservation.granted + BUFFER_FLOOR); + assert!(buffer.bytes.ends_with(suffix)); + assert_eq!(buffer.bytes.len(), 90_002 + suffix.len()); + drop(buffer); + assert_eq!(budget.reserved.load(Ordering::Relaxed), 0); + } + } + + #[test] + fn batch_error_uses_reserved_tail_when_no_more_budget_is_available() { + let budget = ResponseBudget::new(BUDGET_CHUNK); + let mut buffer = ResponseBuffer::new(Vec::new(), Arc::clone(&budget), 100_000); + let delivery = assemble_batch( + vec![json!({"id":1}), json!({"id":"b".repeat(1000)})], + &mut buffer, + &mut |command: &Value| { + if command["id"] == 1 { + json!({"id":1, "result":"a".repeat(80_000)}) + } else { + json!({"id":command["id"], "result":"c".repeat(10_000)}) + } + }, + &mut |limit| assert_eq!(limit, ResponseLimit::Global), + ) + .unwrap(); + assert_eq!(budget.reserved.load(Ordering::Relaxed), BUDGET_CHUNK); + assert!(buffer.bytes.capacity() <= BUDGET_CHUNK + BUFFER_FLOOR); + let mut bytes = Vec::new(); + delivery.send(&mut bytes, &buffer).unwrap(); + let response: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(response[0]["result"].as_str().unwrap().len(), 80_000); + assert_eq!(response[1]["id"], "b".repeat(1000)); + assert_eq!(response[1]["error"]["code"], 2); + drop(buffer); + assert_eq!(budget.reserved.load(Ordering::Relaxed), 0); + } + + #[test] + fn batch_reserves_serialized_errors_before_executing_later_commands() { + let first = json!({"id": 1, "jsonrpc": "2.0", "result": "x".repeat(1000)}); + let second_id = "a\n\"\\é".repeat(10); + let skipped = |id: Value| { + json!({ + "jsonrpc": "2.0", "id": id, + "error": {"code": 1, "message": "batch response budget exhausted; not executed"} + }) + }; + let reserved_line = json!([first, skipped(json!(second_id)), skipped(json!(3))]); + let exact = serde_json::to_vec(&reserved_line).unwrap().len() + 1; + for cap in [exact, exact - 1] { + let mut writer = ResponseWriter::new(cap, ResponseBudget::new(usize::MAX)); + let mut calls = Vec::new(); + let mut bytes = Vec::new(); + writer + .send( + &mut bytes, + Ok(json!([{"id":1}, {"id":second_id}, {"id":3}])), + |cmd| { + calls.push(cmd["id"].clone()); + if cmd["id"] == 1 { + first.clone() + } else { + json!({"id": cmd["id"], "result": "y".repeat(cap)}) + } + }, + |limit| assert_eq!(limit, ResponseLimit::PerLine), + ) + .unwrap(); + let actual: Value = serde_json::from_slice(&bytes).unwrap(); + assert!(bytes.len() <= cap); + assert_eq!(actual[1]["id"], second_id); + assert_eq!(actual[2], skipped(json!(3))); + if cap == exact { + assert_eq!(actual[0], first); + assert_eq!(calls, [json!(1), json!(second_id)]); + assert_eq!(actual[1]["error"]["code"], 1); + } else { + assert_eq!(actual[0]["error"]["code"], 1); + assert_eq!(calls, [json!(1)]); + } + } + } + + #[test] + fn fallback_moves_only_ids_and_streams_without_a_grant() { + for batch in [false, true] { + let id = "x".repeat(32 * 1024); + let command = json!({"id": Value::String(id), "params": ["p".repeat(100_000)]}); + let allocation = command["id"].as_str().unwrap().as_ptr(); + let budget = ResponseBudget::new(1); + let mut buffer = ResponseBuffer::new(Vec::new(), Arc::clone(&budget), 100_000); + let mut execute = |_: &Value| panic!("rejected command executed"); + let mut rejected = |limit| assert_eq!(limit, ResponseLimit::Global); + let delivery = if batch { + assemble_batch(vec![command], &mut buffer, &mut execute, &mut rejected) + } else { + assemble_single(command, &mut buffer, &mut execute, &mut rejected) + } + .unwrap(); + match &delivery { + Delivery::BudgetErrors { + ids, + batch: is_batch, + } => { + assert_eq!(*is_batch, batch); + assert_eq!(ids.len(), 1); + assert_eq!(ids[0].as_str().unwrap().as_ptr(), allocation); + } + Delivery::Buffered => panic!("expected streaming fallback"), + } + assert_eq!(buffer.bytes.capacity(), 0); + assert_eq!(budget.reserved.load(Ordering::Relaxed), 0); + let mut bytes = Vec::new(); + delivery.send(&mut bytes, &buffer).unwrap(); + let response: Value = serde_json::from_slice(&bytes).unwrap(); + let response = if batch { &response[0] } else { &response }; + assert_eq!(response["id"].as_str().unwrap().len(), 32 * 1024); + assert_eq!(response["error"]["code"], 2); + } + } + + #[test] + fn budget_is_shared_until_reservations_drop_and_servers_are_independent() { + let budget = ResponseBudget::new(2 * BUDGET_CHUNK); + let reserve = || Reservation { + budget: Arc::clone(&budget), + granted: 0, + }; + let mut first = reserve(); + let mut second = reserve(); + first.ensure(BUDGET_CHUNK).unwrap(); + second.ensure(BUDGET_CHUNK).unwrap(); + let mut third = reserve(); + assert_eq!(third.ensure(1), Err(ResponseLimit::Global)); + let mut independent = Reservation { + budget: ResponseBudget::new(BUDGET_CHUNK), + granted: 0, + }; + independent.ensure(BUDGET_CHUNK).unwrap(); + drop(first); + third.ensure(BUDGET_CHUNK).unwrap(); + drop(second); + drop(third); + assert_eq!(budget.reserved.load(Ordering::Relaxed), 0); + } + + #[test] + fn concurrent_reservations_never_exceed_the_budget() { + const THREADS: usize = 8; + const GRANTS: usize = 4; + for _ in 0..16 { + let budget = ResponseBudget::new(GRANTS * BUDGET_CHUNK); + let barrier = Arc::new(Barrier::new(THREADS)); + let handles: Vec<_> = (0..THREADS) + .map(|_| { + let budget = Arc::clone(&budget); + let barrier = Arc::clone(&barrier); + std::thread::spawn(move || { + let mut reservation = Reservation { budget, granted: 0 }; + barrier.wait(); + let admitted = reservation.ensure(BUDGET_CHUNK).is_ok(); + // Keep grants alive until every thread has attempted admission. + barrier.wait(); + admitted + }) + }) + .collect(); + let admitted = handles + .into_iter() + .map(|handle| usize::from(handle.join().unwrap())) + .sum::(); + assert_eq!(admitted, GRANTS); + assert_eq!(budget.reserved.load(Ordering::Relaxed), 0); + } + } + + #[test] + fn transmission_keeps_the_grant_and_releases_it_on_error_or_panic() { + struct FailingSink { + budget: Arc, + panic: bool, + } + impl Write for FailingSink { + fn write(&mut self, _: &[u8]) -> io::Result { + assert!(self.budget.reserved.load(Ordering::Relaxed) > 0); + if self.panic { + panic!("simulated writer panic"); + } + Err(io::Error::new(io::ErrorKind::BrokenPipe, "closed")) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + for panic in [false, true] { + let budget = ResponseBudget::new(100_000); + let mut writer = ResponseWriter::new(100_000, Arc::clone(&budget)); + let mut sink = FailingSink { + budget: Arc::clone(&budget), + panic, + }; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + writer.send( + &mut sink, + Ok(json!({"id":1})), + |_| json!({"id":1, "result":"x".repeat(90_000)}), + |_| panic!("unexpected rejection"), + ) + })); + if panic { + assert!(result.is_err()); + } else { + assert!(result.unwrap().is_err()); + } + assert_eq!(budget.reserved.load(Ordering::Relaxed), 0); + assert!(writer.buffer.capacity() <= BUFFER_FLOOR); + } + } + + #[test] + fn parse_errors_empty_batches_and_notifications_keep_their_wire_behavior() { + let mut writer = ResponseWriter::new(128, ResponseBudget::new(1)); + let mut bytes = Vec::new(); + writer + .send( + &mut bytes, + serde_json::from_str("{"), + |_| panic!("parse error executed"), + |_| panic!("unexpected rejection"), + ) + .unwrap(); + let error: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + error, + json!({"jsonrpc":"2.0", "id":null, "error":{"code":-32700,"message":"parse error"}}) + ); + bytes.clear(); + writer + .send( + &mut bytes, + Ok(json!([])), + |_| panic!("empty batch executed"), + |_| panic!("unexpected rejection"), + ) + .unwrap(); + assert_eq!(bytes, b"[]\n"); + bytes.clear(); + let notification = json!({"method":"test", "params":["x".repeat(100_000)]}); + writer + .send_notification(&mut bytes, ¬ification) + .unwrap(); + assert_eq!( + serde_json::from_slice::(&bytes).unwrap(), + notification + ); + assert_eq!(writer.budget.reserved.load(Ordering::Relaxed), 0); + assert!(writer.buffer.capacity() <= BUFFER_FLOOR); + } +} diff --git a/src/electrum/server.rs b/src/electrum/server.rs index 44d2420ed..7115015e4 100644 --- a/src/electrum/server.rs +++ b/src/electrum/server.rs @@ -1,6 +1,6 @@ use std::cmp::Ordering; use std::collections::{BinaryHeap, HashMap}; -use std::io::{BufRead, BufReader, Write}; +use std::io::{BufRead, BufReader}; use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream}; use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender, SyncSender, TrySendError}; use std::sync::{Arc, Mutex, RwLock, Weak}; @@ -21,9 +21,13 @@ use bitcoin::consensus::encode::serialize_hex; use elements::encode::serialize_hex; use crate::chain::Txid; use crate::config::{Config, RpcLogging}; +use crate::electrum::client_io::write_to_client; +use crate::electrum::response::{ + json_rpc_error, warn_limits, JsonRpcV2Error, ResponseBudget, ResponseLimit, ResponseWriter, +}; use crate::electrum::{get_electrum_height, ProtocolVersion}; use crate::errors::*; -use crate::metrics::{Gauge, HistogramOpts, HistogramVec, MetricOpts, Metrics}; +use crate::metrics::{Counter, Gauge, HistogramOpts, HistogramVec, MetricOpts, Metrics}; use crate::new_index::{Query, Utxo}; use crate::util::electrum_merkle::{get_header_merkle_proof, get_id_from_pos, get_tx_merkle_proof}; use crate::util::{create_socket, spawn_thread, BlockId, BoolThen, Channel, FullHash, HeaderEntry}; @@ -31,8 +35,6 @@ use crate::util::{create_socket, spawn_thread, BlockId, BoolThen, Channel, FullH const ELECTRS_VERSION: &str = env!("CARGO_PKG_VERSION"); const PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::new(1, 4); const MAX_HEADERS: usize = 2016; -const MAX_ARRAY_BATCH: usize = 20; - #[cfg(feature = "electrum-discovery")] use crate::electrum::{DiscoveryManager, ServerFeatures}; @@ -82,32 +84,12 @@ fn bool_from_value_or(val: Option<&Value>, name: &str, default: bool) -> Result< bool_from_value(val, name) } -// JSON-RPC 2.0 error codes (https://www.jsonrpc.org/specification#error_object), -// plus the application-level codes used by ElectrumX and romanz/electrs. -#[repr(i16)] -#[derive(Clone, Copy, PartialEq, Eq)] -enum JsonRpcV2Error { - ParseError = -32700, - InvalidRequest = -32600, - MethodNotFound = -32601, - InvalidParams = -32602, - InternalError = -32603, - BadRequest = 1, - DaemonError = 2, -} - -impl JsonRpcV2Error { - #[inline] - fn into_i16(self) -> i16 { - self as i16 - } -} - fn jsonrpc_code(e: &Error) -> JsonRpcV2Error { match e.kind() { ErrorKind::InvalidParams(_) => JsonRpcV2Error::InvalidParams, ErrorKind::TooPopular | ErrorKind::TooManyUtxos + | ErrorKind::TooBigHistory | ErrorKind::TooManySubscriptions(_) => JsonRpcV2Error::BadRequest, // The daemon could not be reached (or we refused to queue for it) for a request // made on the client's behalf. This is a downstream failure, not a client error. @@ -118,22 +100,6 @@ fn jsonrpc_code(e: &Error) -> JsonRpcV2Error { } } -#[inline] -fn json_rpc_error( - input: impl core::fmt::Display, - id: Option<&Value>, - code: JsonRpcV2Error, -) -> Value { - json!({ - "jsonrpc": "2.0", - "id": id.unwrap_or(&Value::Null), - "error": { - "code": code.into_i16(), - "message": format!("{}", input), - }, - }) -} - // TODO: implement caching and delta updates #[trace] fn get_status_hash(txs: Vec<(Txid, Option)>, query: &Query) -> Option { @@ -183,6 +149,9 @@ struct Connection { txs_limit: usize, subscription_limit: usize, max_request_bytes: usize, + max_response_bytes: usize, + write_timeout: Option, + response_budget: Arc, checkpoint_proof_concurrency_limit: usize, #[cfg(feature = "electrum-discovery")] discovery: Option>, @@ -207,6 +176,9 @@ impl Connection { txs_limit: usize, subscription_limit: usize, max_request_bytes: usize, + max_response_bytes: usize, + write_timeout: Option, + response_budget: Arc, checkpoint_proof_concurrency_limit: usize, #[cfg(feature = "electrum-discovery")] discovery: Option>, rpc_logging: RpcLogging, @@ -223,6 +195,9 @@ impl Connection { txs_limit, subscription_limit, max_request_bytes, + max_response_bytes, + write_timeout, + response_budget, checkpoint_proof_concurrency_limit, #[cfg(feature = "electrum-discovery")] discovery, @@ -302,22 +277,33 @@ impl Connection { let height = usize_from_value(params.get(0), "height")?; let cp_height = usize_from_value_or(params.get(1), "cp_height", 0)?; - let raw_header_hex: String = self - .query - .chain() - .header_by_height(height) - .map(|entry| serialize_hex(entry.header())) - .chain_err(|| "missing header")?; - if cp_height == 0 { + let raw_header_hex: String = self + .query + .chain() + .header_by_height(height) + .map(|entry| serialize_hex(entry.header())) + .chain_err(|| "missing header")?; return Ok(json!(raw_header_hex)); } - let (branch, root) = get_header_merkle_proof( + let (branch, root, proof_cp_hash) = get_header_merkle_proof( self.query.chain(), height, cp_height, self.checkpoint_proof_concurrency_limit, )?; + let indexed_headers = self.query.chain().store().headers(); + ensure!( + indexed_headers + .header_by_height(cp_height) + .map(|entry| entry.hash()) + == Some(&proof_cp_hash), + "chain changed while creating checkpoint merkle proof" + ); + let raw_header_hex = indexed_headers + .header_by_height(height) + .map(|entry| serialize_hex(entry.header())) + .chain_err(|| "missing header")?; Ok(json!({ "header": raw_header_hex, @@ -330,18 +316,18 @@ impl Connection { let start_height = usize_from_value(params.get(0), "start_height")?; let count = MAX_HEADERS.min(usize_from_value(params.get(1), "count")?); let cp_height = usize_from_value_or(params.get(2), "cp_height", 0)?; - let heights: Vec = (start_height..(start_height + count)).collect(); - let headers: Vec = heights - .into_iter() - .filter_map(|height| { - self.query - .chain() - .header_by_height(height) - .map(|entry| serialize_hex(entry.header())) - }) - .collect(); - + let end_height = start_height + .checked_add(count) + .chain_err(|| "start_height + count overflows")?; if count == 0 || cp_height == 0 { + let headers: Vec = (start_height..end_height) + .filter_map(|height| { + self.query + .chain() + .header_by_height(height) + .map(|entry| serialize_hex(entry.header())) + }) + .collect(); return Ok(json!({ "count": headers.len(), "hex": headers.join(""), @@ -349,12 +335,27 @@ impl Connection { })); } - let (branch, root) = get_header_merkle_proof( + let (branch, root, proof_cp_hash) = get_header_merkle_proof( self.query.chain(), - start_height + (count - 1), + end_height - 1, cp_height, self.checkpoint_proof_concurrency_limit, )?; + let indexed_headers = self.query.chain().store().headers(); + ensure!( + indexed_headers + .header_by_height(cp_height) + .map(|entry| entry.hash()) + == Some(&proof_cp_hash), + "chain changed while creating checkpoint merkle proof" + ); + let headers: Vec = (start_height..end_height) + .filter_map(|height| { + indexed_headers + .header_by_height(height) + .map(|entry| serialize_hex(entry.header())) + }) + .collect(); Ok(json!({ "count": headers.len(), @@ -415,10 +416,10 @@ impl Connection { #[cfg(not(feature = "liquid"))] fn blockchain_scripthash_get_balance(&self, params: &[Value]) -> Result { let script_hash = hash_from_value(params.get(0))?; - let (chain_stats, mempool_stats) = self.query.stats(&script_hash[..]); + let (chain_stats, mempool_stats) = self.query.stats(&script_hash[..])?; Ok(json!({ - "confirmed": chain_stats.funded_txo_sum - chain_stats.spent_txo_sum, + "confirmed": chain_stats.funded_txo_sum.saturating_sub(chain_stats.spent_txo_sum), "unconfirmed": mempool_stats.funded_txo_sum as i64 - mempool_stats.spent_txo_sum as i64, })) } @@ -710,59 +711,60 @@ impl Connection { println!("{}", log); } - fn send_values(&mut self, values: &[Value]) -> Result<()> { - for value in values { - let line = value.to_string() + "\n"; - (&*self.stream) - .write_all(line.as_bytes()) - .chain_err(|| format!("failed to send response ({} bytes)", line.len()))?; - } - Ok(()) - } - #[trace] fn handle_replies(&mut self, receiver: Receiver) -> Result<()> { let empty_params = json!([]); + let stream = Arc::clone(&self.stream); + let stats = Arc::clone(&self.stats); + let write_timeout = self.write_timeout; + let mut response_writer = + ResponseWriter::new(self.max_response_bytes, Arc::clone(&self.response_budget)); loop { let msg = receiver.recv().chain_err(|| "channel closed")?; trace!("RPC {:?}", msg); match msg { Message::Request(line) => { - let reply = match from_str::(&line) { - Ok(Value::Array(arr)) => { - if arr.len() > MAX_ARRAY_BATCH { - bail!( - "Too many elements in batch requests {} max:{}", - arr.len(), - MAX_ARRAY_BATCH - ); - } - let mut result = Vec::with_capacity(arr.len()); - for el in arr { - result.push(self.handle_value(el, &empty_params)); - } - Value::Array(result) - } - Ok(cmd) => self.handle_value(cmd, &empty_params), - Err(err) => { - warn!("[{}] invalid JSON request: {}", self.addr, err); - json_rpc_error("parse error", None, JsonRpcV2Error::ParseError) - } - }; - self.send_values(&[reply])? + let request = from_str::(&line); + drop(line); + if let Err(error) = &request { + warn!("[{}] invalid JSON request: {}", self.addr, error); + } + write_to_client( + &stream, + write_timeout, + &stats.client_write_timeouts, + |writer| { + response_writer.send( + writer, + request, + |command| self.handle_value(command, &empty_params), + |limit| match limit { + ResponseLimit::PerLine => stats.per_line_overflows.inc(), + ResponseLimit::Global => stats.global_overflows.inc(), + }, + ) + }, + )?; } Message::PeriodicUpdate => { let values = self .update_subscriptions() .chain_err(|| "failed to update subscriptions")?; - self.send_values(&values)? + for value in &values { + write_to_client( + &stream, + write_timeout, + &stats.client_write_timeouts, + |writer| response_writer.send_notification(writer, value), + )?; + } } Message::Done => return Ok(()), } } } - fn handle_value(&mut self, cmd: Value, empty_params: &Value) -> Value { + fn handle_value(&mut self, cmd: &Value, empty_params: &Value) -> Value { let start_time = Instant::now(); match ( cmd.get("method"), @@ -893,7 +895,12 @@ impl Connection { Connection::reader_thread(reader, sender, max_request_bytes) }); if let Err(e) = self.handle_replies(receiver) { - if is_disconnect(&e) { + if matches!(e.kind(), ErrorKind::ClientWriteTimeout) { + warn!( + "[{}] client response write timed out; closing connection", + self.addr + ); + } else if is_disconnect(&e) { // client went away mid-exchange (broken pipe / reset) — not actionable debug!("[{}] connection closed by client: {}", self.addr, e); } else { @@ -1129,6 +1136,9 @@ struct Stats { latency: HistogramVec, clients: Gauge, subscriptions: Gauge, + per_line_overflows: Counter, + global_overflows: Counter, + client_write_timeouts: Counter, } impl RPC { @@ -1221,6 +1231,18 @@ impl RPC { "electrum_subscriptions", "# of Electrum subscriptions", )), + per_line_overflows: metrics.counter(MetricOpts::new( + "electrum_per_line_response_overflows", + "Replies rejected for exceeding the per-line response cap", + )), + global_overflows: metrics.counter(MetricOpts::new( + "electrum_global_response_budget_rejections", + "Replies rejected because the global response budget was exhausted", + )), + client_write_timeouts: metrics.counter(MetricOpts::new( + "electrum_client_write_timeouts_total", + "Connections closed because a client response write timed out", + )), }); stats.clients.set(0); stats.subscriptions.set(0); @@ -1255,9 +1277,14 @@ impl RPC { let txs_limit = config.electrum_txs_limit; let subscription_limit = config.electrum_subscription_limit; let max_request_bytes = config.electrum_rpc_max_request_num_bytes; + let max_response_bytes = config.electrum_rpc_max_response_num_bytes; + let write_timeout = config.electrum_rpc_write_timeout; + let response_budget = ResponseBudget::new(config.electrum_rpc_global_response_budget_bytes); let checkpoint_proof_concurrency_limit = config.electrum_checkpoint_proof_concurrency_limit; let conn_max_age = config.electrum_rpc_conn_max_age; + warn_limits(max_request_bytes, max_response_bytes, txs_limit); + RPC { notification: notification.sender(), server: Some(spawn_thread("rpc", move || { @@ -1286,6 +1313,7 @@ impl RPC { while let Some((stream, addr)) = acceptor.receiver().recv().unwrap() { // explicitly scope the shadowed variables for the new thread let query = Arc::clone(&query); + let response_budget = Arc::clone(&response_budget); let stats = Arc::clone(&stats); let garbage_sender = garbage_sender.clone(); let rpc_logging = config.rpc_logging.clone(); @@ -1308,6 +1336,9 @@ impl RPC { txs_limit, subscription_limit, max_request_bytes, + max_response_bytes, + write_timeout, + response_budget, checkpoint_proof_concurrency_limit, #[cfg(feature = "electrum-discovery")] discovery, @@ -1373,6 +1404,7 @@ impl Drop for RPC { #[cfg(test)] mod tests { use super::*; + use std::io::Write; #[test] fn test_hash_ip_with_salt() { diff --git a/src/elements/mod.rs b/src/elements/mod.rs index e0d044c2b..957dc50d0 100644 --- a/src/elements/mod.rs +++ b/src/elements/mod.rs @@ -97,10 +97,14 @@ pub mod ebcompact { pub trait TxidCompat { fn compute_txid(&self) -> elements::Txid; + fn compute_wtxid(&self) -> elements::Wtxid; } impl TxidCompat for elements::Transaction { fn compute_txid(&self) -> elements::Txid { self.txid() } + fn compute_wtxid(&self) -> elements::Wtxid { + self.wtxid() + } } } diff --git a/src/errors.rs b/src/errors.rs index c8d09fc2e..bd538207b 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -3,6 +3,10 @@ error_chain! { Error, ErrorKind, ResultExt, Result; } + foreign_links { + RocksDb(rocksdb::Error); + } + errors { Connection(msg: String) { description("Connection error") @@ -29,11 +33,21 @@ error_chain! { display("Too many unspent outputs") } + TooBigHistory { + description("Scripthash history too large to scan") + display("Scripthash history too large to scan") + } + TooManySubscriptions(limit: usize) { description("Too many subscriptions") display("Too many subscriptions on this connection (limit: {})", limit) } + MissingTxo(outpoint: String) { + description("Missing transaction output") + display("missing txo {}", outpoint) + } + InvalidParams(msg: String) { description("Invalid RPC params") display("{}", msg) @@ -55,6 +69,21 @@ error_chain! { display("Daemon is unavailable: {}", msg) } + PerLineResponseOverflow { + description("Response line exceeds configured cap") + display("response line exceeds max_response_bytes") + } + + ResponseBudgetExhausted { + description("Global response budget exhausted") + display("response budget exhausted, retry later") + } + + ClientWriteTimeout { + description("Client response write timed out") + display("client response write timed out") + } + #[cfg(feature = "electrum-discovery")] ElectrumClient(e: electrum_client::Error) { description("Electrum client error") diff --git a/src/lib.rs b/src/lib.rs index af632d973..1d6fb56e0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,16 @@ extern crate serde_json; #[macro_use] extern crate lazy_static; +// Reuse the integration-test node fixture in private REST unit tests. +#[cfg(test)] +extern crate self as electrs; + +#[cfg(test)] +#[allow(dead_code)] +mod test_common { + include!(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/common.rs")); +} + pub mod chain; pub mod config; pub mod daemon; diff --git a/src/metrics.rs b/src/metrics.rs index e0c56ac75..e32dab068 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -72,6 +72,11 @@ impl Metrics { h } + /// Snapshot of every registered metric, e.g. for asserting on counters in tests. + pub fn gather(&self) -> Vec { + self.reg.gather() + } + pub fn start(&self) { let server = tiny_http::Server::http(self.addr) .unwrap_or_else(|_| panic!("failed to start monitoring HTTP server at {}", self.addr)); diff --git a/src/new_index/db.rs b/src/new_index/db.rs index 73193fc7f..9cac4e8df 100644 --- a/src/new_index/db.rs +++ b/src/new_index/db.rs @@ -360,6 +360,10 @@ impl DB { self.db.get(key).unwrap().map(|v| v.to_vec()) } + pub fn try_get(&self, key: &[u8]) -> Result, rocksdb::Error> { + self.db.get(key).map(|value| value.map(|v| v.to_vec())) + } + pub fn multi_get(&self, keys: I) -> Vec>, rocksdb::Error>> where K: AsRef<[u8]>, @@ -376,14 +380,53 @@ impl DB { } fn verify_compatibility(&self, config: &Config) { - let compatibility_bytes = bincode::serialize_little(&(DB_VERSION, config.light_mode)).unwrap(); + let version_bytes = bincode::serialize_little(&(DB_VERSION, config.light_mode)).unwrap(); + let flags_bytes = + bincode::serialize_little(&(config.index_unspendables, config.address_search)) + .unwrap(); + // Kept apart from "V" so binaries that only know (DB_VERSION, light_mode) can still open the DB. + const FLAGS_KEY: &[u8] = b"VF"; + + let stored_version = match self.get(b"V") { + None => { + self.put(FLAGS_KEY, &flags_bytes); + return self.put(b"V", &version_bytes); + } + Some(stored) => stored, + }; - match self.get(b"V") { - None => self.put(b"V", &compatibility_bytes), - Some(x) if x != compatibility_bytes => { + if stored_version != version_bytes { + let combined_bytes = bincode::serialize_little(&( + DB_VERSION, + config.light_mode, + config.index_unspendables, + config.address_search, + )) + .unwrap(); + if stored_version != combined_bytes { panic!("Incompatible database found. Please reindex or migrate.") } - Some(_) => (), + self.put(FLAGS_KEY, &flags_bytes); + return self.put(b"V", &version_bytes); + } + + match self.get(FLAGS_KEY) { + Some(stored_flags) if stored_flags == flags_bytes => {} + Some(_) => panic!( + "Incompatible database found: index_unspendables={} and address_search={} \ + differ from the settings used to build this database. Please reindex.", + config.index_unspendables, config.address_search, + ), + None => { + warn!( + "recording DB index flags index_unspendables={} and address_search={}; \ + these values must match the settings originally used to build this database, and \ + historical values cannot be verified", + config.index_unspendables, + config.address_search, + ); + self.put(FLAGS_KEY, &flags_bytes); + } } } @@ -595,4 +638,170 @@ mod tests { } assert_eq!(count, 9, "expected 9 total rows, got {}", count); } + + fn make_test_config(index_unspendables: bool, address_search: bool, light_mode: bool) -> crate::config::Config { + use crate::config::{Config, RpcLogging}; + use std::path::PathBuf; + + #[cfg(not(feature = "liquid"))] + let network_type = crate::chain::Network::Regtest; + #[cfg(feature = "liquid")] + let network_type = crate::chain::Network::LiquidRegtest; + + let unused_addr = "127.0.0.1:0".parse().unwrap(); + + Config { + log: stderrlog::new(), + network_type, + db_path: PathBuf::from("/tmp/electrs-test-unused"), + daemon_dir: PathBuf::from("/tmp/electrs-test-unused"), + blocks_dir: PathBuf::from("/tmp/electrs-test-unused"), + daemon_rpc_addr: unused_addr, + daemon_rpc_fallback_addr: None, + daemon_parallelism: 1, + daemon_conn_max_age: None, + cookie: None, + electrum_rpc_addr: unused_addr, + electrum_rpc_conn_max_age: None, + electrum_rpc_write_timeout: Some(std::time::Duration::from_secs(30)), + electrum_rpc_max_request_num_bytes: 1_048_576, + electrum_rpc_max_response_num_bytes: 8_388_608, + electrum_rpc_global_response_budget_bytes: 67_108_864, + http_addr: unused_addr, + http_socket_file: None, + monitoring_addr: unused_addr, + jsonrpc_import: false, + light_mode, + address_search, + index_unspendables, + enable_mining_rest: false, + cors: None, + precache_scripts: None, + utxos_limit: 100, + history_scan_limit: 100_000, + utxos_checkpoint_limit: 1_000, + electrum_txs_limit: 100, + electrum_subscription_limit: 10_000, + electrum_checkpoint_proof_concurrency_limit: 2, + electrum_checkpoint_merkle_cache_mb: 256, + electrum_banner: String::new(), + rpc_logging: RpcLogging::default(), + zmq_addr: None, + db_block_cache_mb: 8, + db_parallelism: 1, + db_write_buffer_size_mb: 8, + initial_sync_batch_size: 10, + db_cache_index_filter_blocks: false, + #[cfg(feature = "liquid")] + parent_network: bitcoin::Network::Regtest, + #[cfg(feature = "liquid")] + asset_db_path: None, + } + } + + #[test] + fn test_verify_compatibility_accepts_matching_flags() { + let dir = tempfile::tempdir().unwrap(); + let db = DB::open_test(dir.path()); + + let config = make_test_config(true, false, false); + db.verify_compatibility(&config); + db.verify_compatibility(&config); + } + + #[test] + fn test_verify_compatibility_rejects_index_unspendables_mismatch() { + let dir = tempfile::tempdir().unwrap(); + let db = DB::open_test(dir.path()); + + db.verify_compatibility(&make_test_config(false, false, false)); + + let toggled = make_test_config(true, false, false); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + db.verify_compatibility(&toggled); + })); + assert!( + result.is_err(), + "verify_compatibility should reject a DB reopened with a different index_unspendables flag" + ); + } + + #[test] + fn test_verify_compatibility_rejects_address_search_mismatch() { + let dir = tempfile::tempdir().unwrap(); + let db = DB::open_test(dir.path()); + + db.verify_compatibility(&make_test_config(false, false, false)); + + let toggled = make_test_config(false, true, false); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + db.verify_compatibility(&toggled); + })); + assert!( + result.is_err(), + "verify_compatibility should reject a DB reopened with a different address_search flag" + ); + } + + #[test] + fn test_verify_compatibility_upgrades_legacy_record_without_reindex() { + let dir = tempfile::tempdir().unwrap(); + let db = DB::open_test(dir.path()); + + let legacy_bytes = bincode::serialize_little(&(DB_VERSION, false)).unwrap(); + db.put(b"V", &legacy_bytes); + + let config = make_test_config(true, true, false); + db.verify_compatibility(&config); + + assert_eq!( + db.get(b"V").unwrap(), + legacy_bytes, + "the version record must stay readable by binaries that predate the flags" + ); + + let toggled = make_test_config(false, true, false); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + db.verify_compatibility(&toggled); + })); + assert!(result.is_err(), "recorded flags should still guard future mismatches"); + } + + #[test] + fn test_verify_compatibility_restores_legacy_record_from_combined_format() { + let dir = tempfile::tempdir().unwrap(); + let db = DB::open_test(dir.path()); + + let combined = bincode::serialize_little(&(DB_VERSION, false, true, false)).unwrap(); + db.put(b"V", &combined); + + db.verify_compatibility(&make_test_config(true, false, false)); + + let legacy_bytes = bincode::serialize_little(&(DB_VERSION, false)).unwrap(); + assert_eq!(db.get(b"V").unwrap(), legacy_bytes); + + let toggled = make_test_config(true, true, false); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + db.verify_compatibility(&toggled); + })); + assert!(result.is_err(), "flags from the combined record must be carried over"); + } + + #[test] + fn test_verify_compatibility_rejects_legacy_record_with_different_light_mode() { + let dir = tempfile::tempdir().unwrap(); + let db = DB::open_test(dir.path()); + + let legacy_bytes = bincode::serialize_little(&(DB_VERSION, true)).unwrap(); + db.put(b"V", &legacy_bytes); + + let config = make_test_config(false, false, false); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + db.verify_compatibility(&config); + })); + assert!( + result.is_err(), + "verify_compatibility should reject a legacy DB with a different light_mode" + ); + } } diff --git a/src/new_index/mempool.rs b/src/new_index/mempool.rs index ddb91fe9c..691795d3d 100644 --- a/src/new_index/mempool.rs +++ b/src/new_index/mempool.rs @@ -1,4 +1,5 @@ use arraydeque::{ArrayDeque, Wrapping}; +use error_chain::ChainedError; use itertools::{Either, Itertools}; #[cfg(not(feature = "liquid"))] @@ -16,7 +17,7 @@ use crate::chain::{deserialize, BlockHash, Network, OutPoint, Transaction, TxOut use crate::config::Config; use crate::daemon::Daemon; use crate::errors::*; -use crate::metrics::{GaugeVec, HistogramOpts, HistogramVec, MetricOpts, Metrics}; +use crate::metrics::{Counter, GaugeVec, HistogramOpts, HistogramVec, MetricOpts, Metrics}; use crate::new_index::{ compute_script_hash, schema::FullHash, ChainQuery, FundingInfo, GetAmountVal, ScriptStats, SpendingInfo, SpendingInput, TxHistoryInfo, Utxo, @@ -30,6 +31,13 @@ use crate::elements::asset; const RECENT_TXS_SIZE: usize = 10; const BACKLOG_STATS_TTL: u64 = 10; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MempoolSyncStatus { + Synced, + TipMoved, + FailedToIndex, +} + pub struct Mempool { chain: Arc, config: Arc, @@ -46,6 +54,7 @@ pub struct Mempool { latency: HistogramVec, // mempool requests latency delta: HistogramVec, // # of added/removed txs count: GaugeVec, // current state of the mempool + sync_failed_to_index: Counter, // # of times a mempool sync failed to index fetched txs // elements only #[cfg(feature = "liquid")] @@ -79,7 +88,9 @@ impl Mempool { recent: ArrayDeque::new(), backlog_stats: ( BacklogStats::default(), - Instant::now() - Duration::from_secs(BACKLOG_STATS_TTL), + Instant::now() + .checked_sub(Duration::from_secs(BACKLOG_STATS_TTL)) + .unwrap_or_else(Instant::now), ), latency: metrics.histogram_vec( HistogramOpts::new("mempool_latency", "Mempool requests latency (in seconds)"), @@ -93,6 +104,10 @@ impl Mempool { MetricOpts::new("mempool_count", "# of elements currently at the mempool"), &["type"], ), + sync_failed_to_index: metrics.counter(MetricOpts::new( + "mempool_sync_failed_to_index", + "# of times a mempool sync failed to index fetched transactions", + )), #[cfg(feature = "liquid")] asset_history: HashMap::new(), @@ -179,14 +194,18 @@ impl Mempool { } #[trace] - pub fn utxo(&self, scripthash: &[u8]) -> Vec { + pub fn utxo(&self, scripthash: &[u8]) -> Result> { let _timer = self.latency.with_label_values(&["utxo"]).start_timer(); let entries = match self.history.get(scripthash) { - None => return vec![], + None => return Ok(vec![]), Some(entries) => entries, }; + ensure!( + entries.len() <= self.config.utxos_checkpoint_limit, + ErrorKind::TooBigHistory + ); - entries + Ok(entries .iter() .filter_map(|entry| match entry { TxHistoryInfo::Funding(info) => { @@ -218,20 +237,24 @@ impl Mempool { | TxHistoryInfo::Pegout(_) => unreachable!(), }) .filter(|utxo| !self.has_spend(&OutPoint::from(utxo))) - .collect() + .collect()) } #[trace] // @XXX avoid code duplication with ChainQuery::stats()? - pub fn stats(&self, scripthash: &[u8]) -> ScriptStats { + pub fn stats(&self, scripthash: &[u8]) -> Result { let _timer = self.latency.with_label_values(&["stats"]).start_timer(); let mut stats = ScriptStats::default(); let mut seen_txids = HashSet::new(); let entries = match self.history.get(scripthash) { - None => return stats, + None => return Ok(stats), Some(entries) => entries, }; + ensure!( + entries.len() <= self.config.utxos_checkpoint_limit, + ErrorKind::TooBigHistory + ); for entry in entries { if seen_txids.insert(entry.get_txid()) { @@ -268,7 +291,7 @@ impl Mempool { }; } - stats + Ok(stats) } #[trace] @@ -307,41 +330,84 @@ impl Mempool { self.backlog_stats = (BacklogStats::from_feeinfo_slice(&feeinfo), Instant::now()); } + /// Ensure a single transaction (e.g. a broadcast one) is in the mempool. Fails if the txid is not + /// already indexed and cannot be fetched from the daemon. #[trace] - pub fn add_by_txid(&mut self, daemon: &Daemon, txid: Txid) -> Result<()> { - if self.txstore.get(&txid).is_none() { - if let Ok(tx) = daemon.getmempooltx(&txid) { - let mut txs_map = HashMap::new(); - txs_map.insert(txid, tx); - self.add(txs_map) - } else { - bail!("add_by_txid cannot find {}", txid); - } - } else { - Ok(()) + pub fn ensure_tx(mempool: &RwLock, daemon: &Daemon, txid: Txid) -> Result<()> { + Self::ensure_txs_with(mempool, daemon, &[txid], HashMap::new()) + } + + /// Ensure multiple transactions (e.g. an accepted package) are in the mempool in a single batch, + /// so that interdependent parent/child txs are linked within the same `add` call. + pub fn ensure_txs(mempool: &RwLock, daemon: &Daemon, txids: &[Txid]) -> Result<()> { + Self::ensure_txs_with(mempool, daemon, txids, HashMap::new()) + } + + /// Like `ensure_txs`, but takes the transactions found in `known` as-is instead of fetching + /// them from the daemon. Fails if any txid could not be fetched or indexed. + pub fn ensure_txs_with( + mempool: &RwLock, + daemon: &Daemon, + txids: &[Txid], + known: HashMap, + ) -> Result<()> { + let unavailable = Self::add_missing(mempool, daemon, txids, known)?; + if !unavailable.is_empty() { + bail!( + "cannot find {} of {} mempool txids first='{}'", + unavailable.len(), + txids.len(), + unavailable[0] + ); } + Ok(()) } - /// Add multiple transactions (e.g. an accepted package) to the mempool in a single batch, - /// so that interdependent parent/child txs are linked within the same `add` call. Txids that - /// are already present or cannot be fetched from the daemon are skipped. - pub fn add_by_txids(&mut self, daemon: &Daemon, txids: &[Txid]) -> Result<()> { + /// Fetch and index the given txids that are not yet in the local mempool, returning the ones + /// that could not be fetched. The daemon RPCs run without holding the mempool lock, so they + /// never block concurrent readers; the write lock is only taken to insert. + fn add_missing( + mempool: &RwLock, + daemon: &Daemon, + txids: &[Txid], + mut known: HashMap, + ) -> Result> { + let missing: Vec = { + let mempool = mempool.read().unwrap_or_else(|e| e.into_inner()); + txids + .iter() + .copied() + .filter(|txid| mempool.txstore.get(txid).is_none()) + .collect() + }; + let mut txs_map = HashMap::new(); - for &txid in txids { - if self.txstore.get(&txid).is_some() { + let mut unavailable = Vec::new(); + for txid in missing { + if let Some(tx) = known.remove(&txid) { + txs_map.insert(txid, tx); continue; } match daemon.getmempooltx(&txid) { Ok(tx) => { txs_map.insert(txid, tx); } - Err(e) => warn!("add_by_txids cannot find txid='{}': e='{}'", txid, e), + Err(e) => { + warn!("cannot find mempool txid='{}': e='{}'", txid, e); + unavailable.push(txid); + } } } - if txs_map.is_empty() { - return Ok(()); + + if !txs_map.is_empty() { + let mut mempool = mempool.write().unwrap_or_else(|e| e.into_inner()); + // update() may have indexed some of these while we were fetching + txs_map.retain(|txid, _| mempool.txstore.get(txid).is_none()); + if !txs_map.is_empty() { + mempool.add(txs_map)?; + } } - self.add(txs_map) + Ok(unavailable) } #[trace] @@ -464,7 +530,8 @@ impl Mempool { Ok(()) } - fn lookup_txo(&self, outpoint: &OutPoint) -> Option { + /// Look up a local mempool output without chain or daemon IO. + pub(crate) fn lookup_txo(&self, outpoint: &OutPoint) -> Option { self.txstore .get(&outpoint.txid) .and_then(|tx| tx.output.get(outpoint.vout as usize).cloned()) @@ -492,6 +559,44 @@ impl Mempool { Ok(txos) } + /// Remove the transactions whose prevouts cannot be found in the batch, the mempool or the + /// chain, along with their in-batch descendants, returning the removed txids. + fn drop_unresolvable(&self, txs_map: &mut HashMap) -> Vec { + let mut dropped: HashSet = txs_map + .iter() + .filter(|(_, tx)| { + let external: BTreeSet = get_prev_outpoints(std::iter::once(*tx)) + .into_iter() + .filter(|prevout| match txs_map.get(&prevout.txid) { + Some(parent) => parent.output.get(prevout.vout as usize).is_none(), + None => true, + }) + .collect(); + self.lookup_txos(external).is_err() + }) + .map(|(txid, _)| *txid) + .collect(); + + let mut frontier = dropped.clone(); + while !frontier.is_empty() { + frontier = txs_map + .iter() + .filter(|(txid, tx)| { + !dropped.contains(*txid) + && tx + .input + .iter() + .any(|txin| frontier.contains(&txin.previous_output.txid)) + }) + .map(|(txid, _)| *txid) + .collect(); + dropped.extend(frontier.iter().copied()); + } + + txs_map.retain(|txid, _| !dropped.contains(txid)); + dropped.into_iter().collect() + } + #[trace] fn remove(&mut self, to_remove: HashSet<&Txid>) { self.delta @@ -520,7 +625,7 @@ impl Mempool { // coexist in the local view. update() itself is safe (evictions // are applied before additions, diffed against one consistent // bitcoind snapshot), but Query::broadcast_raw()/submit_package() - // inject transactions via add_by_txid(s) outside the sync loop - + // inject transactions via ensure_tx(s) outside the sync loop - // broadcasting a replacement while the original is still indexed // makes the later add() clobber the original's `edges` entry. // Evicting the original then finds its entry gone or foreign. @@ -570,14 +675,13 @@ impl Mempool { .map_or_else(|| vec![], |entries| self._history(entries, limit)) } - /// Sync our local view of the mempool with the bitcoind Daemon RPC. If the chain tip moves before - /// the mempool is fetched in full, syncing is aborted and an Ok(false) is returned. + /// Sync our local view of the mempool with the bitcoind Daemon RPC. #[trace] pub fn update( mempool: &Arc>, daemon: &Daemon, tip: &BlockHash, - ) -> Result { + ) -> Result { let (_timer, count) = { let mempool = mempool.read().unwrap(); let timer = mempool.latency.with_label_values(&["update"]).start_timer(); @@ -622,7 +726,7 @@ impl Mempool { .set(new_txids.len() as f64); if new_txids.is_empty() { - return Ok(true); + return Ok(MempoolSyncStatus::Synced); } // Fetch missing transactions from bitcoind @@ -631,7 +735,7 @@ impl Mempool { // Abort if the chain tip moved while fetching transactions if daemon.getbestblockhash()? != *tip { warn!("chain tip moved while updating mempool"); - return Ok(false); + return Ok(MempoolSyncStatus::TipMoved); } // Find which transactions were requested but are no longer available in bitcoind's mempool, @@ -678,10 +782,31 @@ impl Mempool { // Add fetched transactions to our view of the mempool trace!("indexing {} new mempool transactions", fetched_txs.len()); + let mut status = MempoolSyncStatus::Synced; if !fetched_txs.is_empty() { + let fetched_txs_len = fetched_txs.len(); let mut mempool = mempool.write().unwrap(); - mempool.add(fetched_txs)?; + // Query::broadcast_raw()/submit_package() may have indexed some of these via + // ensure_tx(s) since the snapshot above; adding them twice would duplicate + // their history and recent entries. + fetched_txs.retain(|txid, _| mempool.txstore.get(txid).is_none()); + if !fetched_txs.is_empty() { + if let Err(e) = mempool.add(fetched_txs.clone()) { + let unresolvable = mempool.drop_unresolvable(&mut fetched_txs); + warn!( + "failed adding {} fetched txs, skipping {} with unresolvable prevouts until next cycle: {}", + fetched_txs_len, + unresolvable.len(), + e.display_chain() + ); + mempool.sync_failed_to_index.inc(); + if unresolvable.is_empty() || mempool.add(fetched_txs).is_err() { + return Ok(MempoolSyncStatus::FailedToIndex); + } + status = MempoolSyncStatus::FailedToIndex; + } + } count .with_label_values(&["txs"]) @@ -693,9 +818,9 @@ impl Mempool { } } - trace!("mempool is synced"); + trace!("mempool update finished status={:?}", status); - Ok(true) + Ok(status) } } diff --git a/src/new_index/mod.rs b/src/new_index/mod.rs index 8d2a734ff..66bd8ab8d 100644 --- a/src/new_index/mod.rs +++ b/src/new_index/mod.rs @@ -10,7 +10,7 @@ pub mod zmq; pub use self::db::{DBRow, DB}; pub use self::fetch::{BlockEntry, FetchFrom}; -pub use self::mempool::Mempool; +pub use self::mempool::{Mempool, MempoolSyncStatus}; pub use self::query::Query; pub use self::schema::{ compute_script_hash, parse_hash, ChainQuery, FundingInfo, GetAmountVal, Indexer, ScriptStats, diff --git a/src/new_index/precache.rs b/src/new_index/precache.rs index 86c0c6bf2..9154ed7e3 100644 --- a/src/new_index/precache.rs +++ b/src/new_index/precache.rs @@ -2,11 +2,12 @@ use crate::chain::address::Address; use crate::errors::*; use crate::new_index::ChainQuery; use crate::util::FullHash; +use error_chain::ChainedError; use rayon::prelude::*; use bitcoin::hashes::{sha256, Hash}; -use bitcoin::hex::FromHex; +use bitcoin::hex::{DisplayHex, FromHex}; use std::fs::File; use std::io; use std::io::prelude::*; @@ -32,7 +33,21 @@ pub fn precache(chain: &ChainQuery, scripthashes: Vec) { if i % 5 == 0 { info!("running pre-cache for scripthash {}/{}", i + 1, total); } - chain.stats(&scripthash[..]); + // Each TooBigHistory saves a checkpoint further along, so retrying finishes the scan. + loop { + match chain.stats(&scripthash[..]) { + Err(Error(ErrorKind::TooBigHistory, _)) => continue, + Err(e) => { + warn!( + "pre-cache failed scripthash='{}' e='{}'", + scripthash.to_lower_hex_string(), + e.display_chain() + ); + break; + } + Ok(_) => break, + } + } //chain.utxo(&scripthash[..]); }) }); diff --git a/src/new_index/query.rs b/src/new_index/query.rs index 14a851f5c..70a6bb7d9 100644 --- a/src/new_index/query.rs +++ b/src/new_index/query.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, TryLockError}; use std::time::{Duration, Instant}; -use crate::chain::{Network, OutPoint, Transaction, TxOut, Txid}; +use crate::chain::{deserialize, Network, OutPoint, Transaction, Txid, Wtxid}; use crate::config::Config; use crate::daemon::{Daemon, SubmitPackageResult}; use crate::errors::*; @@ -10,7 +10,9 @@ use crate::new_index::block_template::BlockTemplateCache; use crate::new_index::{ChainQuery, Mempool, ScriptStats, SpendingInput, Utxo}; use crate::util::{is_spendable, BlockId, Bytes, TransactionStatus}; +use bitcoin::hex::FromHex; use electrs_macros::trace; +use error_chain::ChainedError; use hyper::body::Bytes as BodyBytes; #[cfg(feature = "liquid")] @@ -74,20 +76,30 @@ impl Query { } pub fn mempool(&self) -> RwLockReadGuard<'_, Mempool> { - self.mempool.read().unwrap() + self.mempool.read().unwrap_or_else(|e| e.into_inner()) } #[trace] pub fn broadcast_raw(&self, txhex: &str) -> Result { let txid = self.daemon.broadcast_raw(txhex)?; - let _ = self - .mempool - .write() - .unwrap() - .add_by_txid(&self.daemon, txid); + // sendrawtransaction also succeeds when a same-txid tx with a different witness is + // already in the mempool, so the submitted hex may not be the daemon's copy. + self.index_broadcast(&[txid], HashMap::new()); Ok(txid) } + /// Index accepted transactions locally, taking those in `known` as-is and fetching the rest + /// from the daemon. The daemon already accepted them, so failures are logged, not returned. + fn index_broadcast(&self, accepted_txids: &[Txid], known: HashMap) { + if let Err(e) = Mempool::ensure_txs_with(&self.mempool, &self.daemon, accepted_txids, known) + { + warn!( + "accepted transactions not yet indexed locally, visible after the next mempool sync: {}", + e.display_chain() + ); + } + } + #[trace] pub fn submit_package( &self, @@ -95,16 +107,15 @@ impl Query { maxfeerate: Option, maxburnamount: Option, ) -> Result { - let result = self.daemon.submit_package(txhex, maxfeerate, maxburnamount)?; + let result = self + .daemon + .submit_package(txhex.clone(), maxfeerate, maxburnamount)?; // Add accepted txs to the local mempool so subscription status updates reflect them // immediately (they read from the local mempool), mirroring broadcast_raw() above. let accepted_txids = result.accepted_txids(); if !accepted_txids.is_empty() { - let _ = self - .mempool - .write() - .unwrap() - .add_by_txids(&self.daemon, &accepted_txids); + let known = submitted_txs(&txhex, &result.accepted_as_submitted()); + self.index_broadcast(&accepted_txids, known); } Ok(result) } @@ -125,7 +136,11 @@ impl Query { let mut utxos = self.chain.utxo(scripthash, self.config.utxos_limit)?; let mempool = self.mempool(); utxos.retain(|utxo| !mempool.has_spend(&OutPoint::from(utxo))); - utxos.extend(mempool.utxo(scripthash)); + utxos.extend(mempool.utxo(scripthash)?); + ensure!( + utxos.len() <= self.config.utxos_limit, + ErrorKind::TooManyUtxos + ); Ok(utxos) } @@ -144,18 +159,19 @@ impl Query { confirmed_txids.chain(mempool_txids).collect() } - pub fn stats(&self, scripthash: &[u8]) -> (ScriptStats, ScriptStats) { - ( - self.chain.stats(scripthash), - self.mempool().stats(scripthash), - ) + pub fn stats(&self, scripthash: &[u8]) -> Result<(ScriptStats, ScriptStats)> { + Ok(( + self.chain.stats(scripthash)?, + self.mempool().stats(scripthash)?, + )) } #[trace] - pub fn lookup_txn(&self, txid: &Txid) -> Option { - self.chain - .lookup_txn(txid, None) - .or_else(|| self.mempool().lookup_txn(txid)) + pub fn lookup_txn(&self, txid: &Txid) -> Result> { + Ok(self + .chain + .lookup_txn(txid, None)? + .or_else(|| self.mempool().lookup_txn(txid))) } #[trace] @@ -165,14 +181,6 @@ impl Query { .or_else(|| self.mempool().lookup_raw_txn(txid)) } - #[trace] - pub fn lookup_txos(&self, outpoints: BTreeSet) -> HashMap { - // the mempool lookup_txos() internally looks up confirmed txos as well - self.mempool() - .lookup_txos(outpoints) - .expect("failed loading txos") - } - #[trace] pub fn lookup_spend(&self, outpoint: &OutPoint) -> Option { self.chain @@ -356,3 +364,56 @@ impl Query { Ok((total_num, results)) } } + +/// Parse the submitted package transactions that the daemon holds exactly as submitted, matching +/// both txid and wtxid. +fn submitted_txs(txhexes: &[String], submitted: &[(Txid, Wtxid)]) -> HashMap { + txhexes + .iter() + .filter_map(|txhex| Vec::::from_hex(txhex).ok()) + .filter_map(|bytes| deserialize::(&bytes).ok()) + .filter(|tx| submitted.contains(&(tx.compute_txid(), tx.compute_wtxid()))) + .map(|tx| (tx.compute_txid(), tx)) + .collect() +} + +#[cfg(all(test, not(feature = "liquid")))] +mod tests { + use super::submitted_txs; + use bitcoin::consensus::encode::serialize_hex; + use bitcoin::{ + absolute, transaction, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, Witness, + }; + + fn tx_with_witness(item: &[u8]) -> Transaction { + Transaction { + version: transaction::Version::TWO, + lock_time: absolute::LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::from_slice(&[item]), + }], + output: vec![], + } + } + + #[test] + fn submitted_txs_require_matching_txid_and_wtxid() { + let submitted = tx_with_witness(&[1]); + let other = tx_with_witness(&[2]); + let txhexes = vec![serialize_hex(&submitted)]; + + let matching = [(submitted.compute_txid(), submitted.compute_wtxid())]; + assert_eq!( + submitted_txs(&txhexes, &matching).get(&submitted.compute_txid()), + Some(&submitted) + ); + + let other_witness = [(submitted.compute_txid(), other.compute_wtxid())]; + assert!(submitted_txs(&txhexes, &other_witness).is_empty()); + + assert!(submitted_txs(&txhexes, &[]).is_empty()); + } +} diff --git a/src/new_index/schema.rs b/src/new_index/schema.rs index 2ef6ce74d..a49639513 100644 --- a/src/new_index/schema.rs +++ b/src/new_index/schema.rs @@ -27,8 +27,8 @@ use crate::daemon::Daemon; use crate::errors::*; use crate::metrics::{Gauge, HistogramOpts, HistogramTimer, HistogramVec, MetricOpts, Metrics}; use crate::util::{ - bincode, full_hash, has_prevout, is_spendable, BlockHeaderMeta, BlockId, BlockMeta, - BlockStatus, Bytes, HeaderEntry, HeaderList, ScriptToAddr, + bincode, electrum_merkle::CheckpointMerkleCache, full_hash, has_prevout, is_spendable, + BlockHeaderMeta, BlockId, BlockMeta, BlockStatus, Bytes, HeaderEntry, HeaderList, ScriptToAddr, }; use crate::new_index::db::{DBFlush, DBRow, ReverseScanIterator, ScanIterator, DB}; @@ -195,6 +195,12 @@ impl Store { type UtxoMap = HashMap; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum UtxoScanStop { + ScanLimit, + CheckpointLimit, +} + #[derive(Debug)] pub struct Utxo { pub txid: Txid, @@ -292,6 +298,9 @@ pub struct ChainQuery { light_mode: bool, duration: HistogramVec, network: Network, + history_scan_limit: usize, + utxos_checkpoint_limit: usize, + checkpoint_merkle_cache: CheckpointMerkleCache, } // TODO: &[Block] should be an iterator / a queue. @@ -634,10 +643,17 @@ impl ChainQuery { daemon, light_mode: config.light_mode, network: config.network_type, + history_scan_limit: config.history_scan_limit, + utxos_checkpoint_limit: config.utxos_checkpoint_limit, duration: metrics.histogram_vec( HistogramOpts::new("query_duration", "Index query duration (in seconds)"), &["name"], ), + checkpoint_merkle_cache: CheckpointMerkleCache::new( + config + .electrum_checkpoint_merkle_cache_mb + .saturating_mul(1024 * 1024), + ), } } @@ -649,6 +665,10 @@ impl ChainQuery { &self.store } + pub fn checkpoint_merkle_cache(&self) -> &CheckpointMerkleCache { + &self.checkpoint_merkle_cache + } + fn start_timer(&self, name: &str) -> HistogramTimer { self.duration.with_label_values(&[name]).start_timer() } @@ -667,14 +687,35 @@ impl ChainQuery { } } + pub fn try_get_block_txids(&self, hash: &BlockHash) -> Result>> { + let _timer = self.start_timer("get_block_txids"); + if self.light_mode { + let mut blockinfo = match self.daemon.getblock_raw(hash, 1) { + Ok(blockinfo) => blockinfo, + Err(_) => return Ok(None), + }; + Ok(Some(serde_json::from_value(blockinfo["tx"].take()) + .chain_err(|| "failed to parse block txids")?)) + } else { + self.store + .txstore_db + .try_get(&BlockRow::txids_key(full_hash(&hash[..])))? + .map(|val| bincode::deserialize_little(&val).chain_err(|| "failed to parse block txids")) + .transpose() + } + } + pub fn get_block_txs( &self, hash: &BlockHash, start_index: usize, limit: usize, ) -> Result> { - let txids = self.get_block_txids(hash).chain_err(|| "block not found")?; - ensure!(start_index < txids.len(), "start index out of range"); + let txids = self.try_get_block_txids(hash)?.chain_err(|| "block not found")?; + ensure!( + start_index < txids.len(), + ErrorKind::InvalidParams("start index out of range".to_string()) + ); let txids_with_blockhash = txids .into_iter() @@ -772,12 +813,24 @@ impl ChainQuery { ) } + fn history_iter_scan_reverse_from( + &self, + code: u8, + hash: &[u8], + height: usize, + ) -> ReverseScanIterator<'_> { + self.store.history_db.iter_scan_reverse( + &TxHistoryRow::filter(code, &hash[..]), + &TxHistoryRow::prefix_height(code, &hash[..], height as u32 + 1), + ) + } + pub fn history( &self, scripthash: &[u8], last_seen_txid: Option<&Txid>, limit: usize, - ) -> Vec<(Transaction, BlockId)> { + ) -> Result> { // scripthash lookup self._history(b'H', scripthash, last_seen_txid, limit) } @@ -788,24 +841,54 @@ impl ChainQuery { hash: &[u8], last_seen_txid: Option<&Txid>, limit: usize, - ) -> Vec<(Transaction, BlockId)> { + ) -> Result> { let _timer_scan = self.start_timer("history"); let headers = self.store.indexed_headers.read().unwrap(); - let history_iter = self - .history_iter_scan_reverse(code, hash) + + let seek_height = match last_seen_txid { + Some(txid) => match self.tx_confirming_block_with_headers(txid, &headers) { + Some(blockid) => Some(blockid.height), + None => return Ok(vec![]), + }, + None => None, + }; + + let raw_iter = match seek_height { + Some(height) => self.history_iter_scan_reverse_from(code, hash, height), + None => self.history_iter_scan_reverse(code, hash), + }; + + let mut scanned = 0usize; + let mut hit_scan_limit = false; + let scan_limit = self.history_scan_limit; + let history_iter = raw_iter + .take_while(|_| { + scanned += 1; + if scanned > scan_limit { + hit_scan_limit = true; + false + } else { + true + } + }) .map(TxHistoryRow::from_row) .map(|row| (row.get_txid(), row.key.confirmed_height as usize)) // XXX: unique() requires keeping an in-memory list of all txids, can we avoid that? .unique() - // TODO seek directly to last seen tx without reading earlier rows - .skip_while(|(txid, _)| { - // skip until we reach the last_seen_txid - last_seen_txid.map_or(false, |last_seen_txid| last_seen_txid != txid) - }) - .skip(match last_seen_txid { - Some(_) => 1, // skip the last_seen_txid itself - None => 0, + // skip up to and including last_seen_txid, stopping once the scan passes its height + .scan(last_seen_txid.is_none(), |cursor_seen, (txid, height)| { + if *cursor_seen { + Some(Some((txid, height))) + } else if last_seen_txid == Some(&txid) { + *cursor_seen = true; + Some(None) + } else if seek_height.map_or(false, |seek_height| height < seek_height) { + None + } else { + Some(None) + } }) + .flatten() // skip over entries that point to non-existing heights (may happen while new/reorged blocks are being processed) .filter_map(|(txid, height)| Some((txid, headers.header_by_height(height)?))) .take(limit); @@ -818,12 +901,16 @@ impl ChainQuery { } drop(headers); - self.lookup_txns(&txids_with_blockhash) - .expect("failed looking up txs in history index") + if hit_scan_limit && txids_with_blockhash.len() < limit { + bail!(ErrorKind::TooBigHistory); + } + + Ok(self + .lookup_txns(&txids_with_blockhash)? .into_iter() .zip(blockids) .map(|(tx, blockid)| (tx, blockid)) - .collect() + .collect()) } pub fn history_txids(&self, scripthash: &[u8], limit: usize) -> Vec<(Txid, BlockId)> { @@ -850,10 +937,13 @@ impl ChainQuery { // get the last known utxo set and the blockhash it was updated for. // invalidates the cache if the block was orphaned. - let cache: Option<(UtxoMap, usize)> = self - .store - .cache_db - .get(&UtxoCacheRow::key(scripthash)) + let cached_bytes = self.store.cache_db.get(&UtxoCacheRow::key(scripthash)); + if let Some(ref bytes) = cached_bytes { + if UtxoCacheRow::utxo_count(bytes) > self.utxos_checkpoint_limit { + bail!(ErrorKind::TooBigHistory) + } + } + let cache: Option<(UtxoMap, usize)> = cached_bytes .map(|c| bincode::deserialize_little(&c).unwrap()) .and_then(|(utxos_cache, blockhash)| { self.height_by_hash(&blockhash) @@ -863,14 +953,19 @@ impl ChainQuery { let had_cache = cache.is_some(); // update utxo set with new transactions since - let (newutxos, lastblock, processed_items) = cache.map_or_else( - || self.utxo_delta(scripthash, HashMap::new(), 0, limit), - |(oldutxos, blockheight)| self.utxo_delta(scripthash, oldutxos, blockheight + 1, limit), - )?; + let (newutxos, lastblock, processed_items, stop) = cache.map_or_else( + || self.utxo_delta(scripthash, HashMap::new(), 0), + |(oldutxos, blockheight)| self.utxo_delta(scripthash, oldutxos, blockheight + 1), + ); - // save updated utxo set to cache + if stop == Some(UtxoScanStop::CheckpointLimit) { + bail!(ErrorKind::TooBigHistory) + } + let hit_scan_limit = stop == Some(UtxoScanStop::ScanLimit); + + // save updated utxo set to cache, even if we bail out below if let Some(lastblock) = lastblock { - if had_cache || processed_items > MIN_HISTORY_ITEMS_TO_CACHE { + if had_cache || hit_scan_limit || processed_items > MIN_HISTORY_ITEMS_TO_CACHE { self.store.cache_db.write_rows( vec![UtxoCacheRow::new(scripthash, &newutxos, &lastblock).into_row()], DBFlush::Enable, @@ -878,6 +973,14 @@ impl ChainQuery { } } + if hit_scan_limit { + bail!(ErrorKind::TooBigHistory) + } + + if newutxos.len() > limit { + bail!(ErrorKind::TooManyUtxos) + } + // format as Utxo objects Ok(newutxos .into_iter() @@ -910,8 +1013,7 @@ impl ChainQuery { scripthash: &[u8], init_utxos: UtxoMap, start_height: usize, - limit: usize, - ) -> Result<(UtxoMap, Option, usize)> { + ) -> (UtxoMap, Option, usize, Option) { let _timer = self.start_timer("utxo_delta"); let headers = self.store.indexed_headers.read().unwrap(); let history_iter = self @@ -926,8 +1028,17 @@ impl ChainQuery { let mut utxos = init_utxos; let mut processed_items = 0; let mut lastblock = None; + let mut safe_lastblock = None; + let mut stop = None; for (history, blockid) in history_iter { + if lastblock.is_some_and(|hash| hash != blockid.hash) { + safe_lastblock = lastblock; + if processed_items >= self.history_scan_limit { + stop = Some(UtxoScanStop::ScanLimit); + break; + } + } processed_items += 1; lastblock = Some(blockid.hash); @@ -943,16 +1054,19 @@ impl ChainQuery { | TxHistoryInfo::Pegout(_) => unreachable!(), }; - // abort if the utxo set size excedees the limit at any point in time - if utxos.len() > limit { - bail!(ErrorKind::TooManyUtxos) + if utxos.len() > self.utxos_checkpoint_limit { + return (utxos, None, processed_items, Some(UtxoScanStop::CheckpointLimit)); } } - Ok((utxos, lastblock, processed_items)) + if stop.is_none() { + safe_lastblock = lastblock; + } + + (utxos, safe_lastblock, processed_items, stop) } - pub fn stats(&self, scripthash: &[u8]) -> ScriptStats { + pub fn stats(&self, scripthash: &[u8]) -> Result { let _timer = self.start_timer("stats"); // get the last known stats and the blockhash they are updated for. @@ -968,14 +1082,16 @@ impl ChainQuery { }); // update stats with new transactions since - let (newstats, lastblock) = cache.map_or_else( + let (newstats, lastblock, hit_scan_limit) = cache.map_or_else( || self.stats_delta(scripthash, ScriptStats::default(), 0), |(oldstats, blockheight)| self.stats_delta(scripthash, oldstats, blockheight + 1), ); - // save updated stats to cache + // save updated stats to cache, even if we bail out below if let Some(lastblock) = lastblock { - if newstats.funded_txo_count + newstats.spent_txo_count > MIN_HISTORY_ITEMS_TO_CACHE { + if hit_scan_limit + || newstats.funded_txo_count + newstats.spent_txo_count > MIN_HISTORY_ITEMS_TO_CACHE + { self.store.cache_db.write_rows( vec![StatsCacheRow::new(scripthash, &newstats, &lastblock).into_row()], DBFlush::Enable, @@ -983,7 +1099,11 @@ impl ChainQuery { } } - newstats + if hit_scan_limit { + bail!(ErrorKind::TooBigHistory) + } + + Ok(newstats) } fn stats_delta( @@ -991,7 +1111,7 @@ impl ChainQuery { scripthash: &[u8], init_stats: ScriptStats, start_height: usize, - ) -> (ScriptStats, Option) { + ) -> (ScriptStats, Option, bool) { let _timer = self.start_timer("stats_delta"); // TODO: measure also the number of txns processed. let headers = self.store.indexed_headers.read().unwrap(); let history_iter = self @@ -1006,11 +1126,20 @@ impl ChainQuery { let mut stats = init_stats; let mut seen_txids = HashSet::new(); let mut lastblock = None; + let mut safe_lastblock = None; + let mut processed_items = 0; + let mut hit_scan_limit = false; for (history, blockid) in history_iter { if lastblock != Some(blockid.hash) { + safe_lastblock = lastblock; seen_txids.clear(); + if processed_items >= self.history_scan_limit { + hit_scan_limit = true; + break; + } } + processed_items += 1; if seen_txids.insert(history.get_txid()) { stats.tx_count += 1; @@ -1049,7 +1178,11 @@ impl ChainQuery { lastblock = Some(blockid.hash); } - (stats, lastblock) + if !hit_scan_limit { + safe_lastblock = lastblock; + } + + (stats, safe_lastblock, hit_scan_limit) } pub fn address_search(&self, prefix: &str, limit: usize) -> Vec { @@ -1137,17 +1270,21 @@ impl ChainQuery { pub fn lookup_txns(&self, txids: &[(Txid, BlockHash)]) -> Result> { let _timer = self.start_timer("lookup_txns"); - Ok(self - .lookup_raw_txns(txids)? + self.lookup_raw_txns(txids)? .into_iter() - .map(|rawtx| deserialize(&rawtx).expect("failed to parse Transaction")) - .collect()) + .map(|rawtx| deserialize(&rawtx).chain_err(|| "failed to parse Transaction")) + .collect() } - pub fn lookup_txn(&self, txid: &Txid, blockhash: Option<&BlockHash>) -> Option { + pub fn lookup_txn( + &self, + txid: &Txid, + blockhash: Option<&BlockHash>, + ) -> Result> { let _timer = self.start_timer("lookup_txn"); - let rawtx = self.lookup_raw_txn(txid, blockhash)?; - Some(deserialize(&rawtx).expect("failed to parse Transaction")) + self.try_lookup_raw_txn(txid, blockhash)? + .map(|rawtx| deserialize(&rawtx).chain_err(|| "failed to parse Transaction")) + .transpose() } pub fn lookup_raw_txns(&self, txids: &[(Txid, BlockHash)]) -> Result> { @@ -1166,7 +1303,7 @@ impl ChainQuery { .txstore_db .multi_get(keys) .into_iter() - .map(|val| val.unwrap().chain_err(|| "missing tx")) + .map(|val| val?.chain_err(|| "missing tx")) .collect() } } @@ -1190,6 +1327,28 @@ impl ChainQuery { } } + fn try_lookup_raw_txn(&self, txid: &Txid, blockhash: Option<&BlockHash>) -> Result> { + if self.light_mode { + let queried_blockhash = if blockhash.is_none() { + self.try_tx_confirming_block(txid)?.map(|blockid| blockid.hash) + } else { + None + }; + let blockhash = match blockhash.or(queried_blockhash.as_ref()) { + Some(blockhash) => blockhash, + None => return Ok(None), + }; + let txval = match self.daemon.gettransaction_raw(txid, blockhash, false) { + Ok(txval) => txval, + Err(_) => return Ok(None), + }; + let txhex = txval.as_str().chain_err(|| "invalid transaction RPC response")?; + Ok(Some(Bytes::from_hex(txhex).chain_err(|| "invalid transaction hex")?)) + } else { + Ok(self.store.txstore_db.try_get(&TxRow::key(&txid[..]))?) + } + } + pub fn lookup_txo(&self, outpoint: &OutPoint) -> Option { let _timer = self.start_timer("lookup_txo"); lookup_txo(&self.store.txstore_db, outpoint) @@ -1239,13 +1398,29 @@ impl ChainQuery { pub fn tx_confirming_block(&self, txid: &Txid) -> Option { let _timer = self.start_timer("tx_confirming_block"); + let headers = self.store.indexed_headers.read().unwrap(); + self.tx_confirming_block_with_headers(txid, &headers) + } + + fn tx_confirming_block_with_headers(&self, txid: &Txid, headers: &HeaderList) -> Option { let row_value = self.store.history_db.get(&TxConfRow::key(txid))?; let height = TxConfRow::height_from_val(&row_value); - let headers = self.store.indexed_headers.read().unwrap(); // skip over entries that point to non-existing heights (may happen while new/reorged blocks are being processed) Some(headers.header_by_height(height as usize)?.into()) } + pub fn try_tx_confirming_block(&self, txid: &Txid) -> Result> { + let _timer = self.start_timer("tx_confirming_block"); + let row_value = match self.store.history_db.try_get(&TxConfRow::key(txid))? { + Some(row_value) => row_value, + None => return Ok(None), + }; + ensure!(row_value.len() == 4, "invalid transaction confirmation row"); + let height = TxConfRow::height_from_val(&row_value); + let headers = self.store.indexed_headers.read().unwrap(); + Ok(headers.header_by_height(height as usize).map(Into::into)) + } + pub fn lookup_confirmations(&self, txids: BTreeSet) -> HashMap { let _timer = self.start_timer("lookup_confirmations"); let headers = self.store.indexed_headers.read().unwrap(); @@ -1292,7 +1467,7 @@ impl ChainQuery { asset_id: &AssetId, last_seen_txid: Option<&Txid>, limit: usize, - ) -> Vec<(Transaction, BlockId)> { + ) -> Result> { self._history(b'I', &asset_id.into_inner()[..], last_seen_txid, limit) } @@ -1390,10 +1565,9 @@ fn lookup_txos(txstore_db: &DB, outpoints: BTreeSet) -> Result usize { + value + .get(..8) + .and_then(|prefix| bincode::deserialize_little::(prefix).ok()) + .map_or(0, |count| std::convert::TryFrom::try_from(count).unwrap_or(usize::MAX)) + } + fn into_row(self) -> DBRow { DBRow { key: bincode::serialize_little(&self.key).unwrap(), @@ -2073,6 +2256,93 @@ pub mod bench { #[cfg(test)] mod tests { use super::*; + use crate::test_common as common; + + #[cfg(not(feature = "liquid"))] + #[test] + fn partial_mempool_sync_reports_failure_and_keeps_resolvable_txs() -> common::Result<()> { + use crate::new_index::{Mempool, MempoolSyncStatus}; + use bitcoin::consensus::encode::serialize_hex; + use serde_json::{json, Value}; + + let mut tester = common::TestRunner::new()?; + let address = tester.newaddress()?; + let funding_txid = tester.send_multi(&address, bitcoin::Amount::ONE_BTC, 2)?; + tester.mine()?; + + let spend = |vout: u32| -> common::Result { + let tx = Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::ZERO, + input: vec![bitcoin::TxIn { + previous_output: OutPoint::new(funding_txid, vout), + ..Default::default() + }], + output: vec![TxOut { + value: bitcoin::Amount::from_sat(99_990_000), + script_pubkey: tester.newaddress()?.script_pubkey(), + }], + }; + let signed: Value = tester + .node_client() + .call("signrawtransactionwithwallet", &[json!(serialize_hex(&tx))])?; + Ok(tester + .node_client() + .call("sendrawtransaction", &[signed["hex"].clone()])?) + }; + let unresolvable_txid = spend(0)?; + let resolvable_txid = spend(1)?; + + tester.query().chain().store.txstore_db.delete_rows( + vec![DBRow { + key: TxOutRow::key(&OutPoint::new(funding_txid, 0)), + value: vec![], + }], + DBFlush::Enable, + ); + + let tip = tester.get_best_block_hash()?; + assert_eq!( + Mempool::update(tester.mempool(), tester.daemon(), &tip)?, + MempoolSyncStatus::FailedToIndex + ); + let mempool = tester.mempool().read().unwrap(); + assert!(mempool.lookup_txn(&resolvable_txid).is_some()); + assert!(mempool.lookup_txn(&unresolvable_txid).is_none()); + Ok(()) + } + + #[test] + fn corrupt_single_key_rows_return_errors() -> common::Result<()> { + let mut tester = common::TestRunner::new()?; + let address = tester.newaddress()?; + let txid = tester.send(&address, "1 BTC".parse().unwrap())?; + let blockhash = tester.mine()?; + let chain = tester.query().chain(); + + chain.store.txstore_db.put(&BlockRow::txids_key(full_hash(&blockhash[..])), &[]); + assert!(chain.get_block_txs(&blockhash, 0, 25).is_err()); + + chain.store.history_db.put(&TxConfRow::key(&txid), &[]); + assert!(chain.try_tx_confirming_block(&txid).is_err()); + Ok(()) + } + + #[cfg(not(feature = "liquid"))] + #[test] + fn utxo_cache_row_count_reads_the_serialized_length() { + let blockid = BlockId { + height: 7, + hash: BlockHash::all_zeros(), + time: 0, + }; + let utxos: UtxoMap = (0..3u32) + .map(|vout| (OutPoint::new(Txid::all_zeros(), vout), (blockid, 1_000))) + .collect(); + let row = UtxoCacheRow::new(&[0u8; 32], &utxos, &BlockHash::all_zeros()); + assert_eq!(UtxoCacheRow::utxo_count(&row.value), 3); + assert_eq!(UtxoCacheRow::utxo_count(&[]), 0); + } #[test] fn test_completed_prefix_stops_at_first_gap() { diff --git a/src/rest.rs b/src/rest.rs index 5493a0fc0..650b68abc 100644 --- a/src/rest.rs +++ b/src/rest.rs @@ -4,7 +4,7 @@ use crate::chain::{ }; use crate::config::Config; use crate::errors; -use crate::new_index::{compute_script_hash, Query, SpendingInput, Utxo}; +use crate::new_index::{compute_script_hash, Mempool, Query, SpendingInput, Utxo}; #[cfg(feature = "liquid")] use crate::util::optional_value_for_newer_blocks; use crate::util::{ @@ -38,7 +38,7 @@ use { }; use serde::Serialize; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::num::ParseIntError; use std::os::unix::fs::FileTypeExt; use std::sync::Arc; @@ -479,19 +479,143 @@ impl Default for SpendingValue { fn ttl_by_depth(height: Option, query: &Query) -> u32 { height.map_or(TTL_SHORT, |height| { - if query.chain().best_height() - height >= CONF_FINAL { - TTL_LONG - } else { - TTL_SHORT - } + ttl_by_confirmations(query.chain().best_height(), height) }) } -fn prepare_txs( +fn ttl_by_confirmations(best_height: usize, height: usize) -> u32 { + if best_height.saturating_sub(height) >= CONF_FINAL { + TTL_LONG + } else { + TTL_SHORT + } +} + +// Own the mempool data needed by a response. Chain IO and response construction +// happen after capture returns and releases the read guard. +struct MempoolTxs { txs: Vec<(Transaction, Option)>, + prevouts: HashMap, +} + +impl MempoolTxs { + fn capture(query: &Query, select: impl FnOnce(&Mempool) -> Vec) -> Self { + let mempool = query.mempool(); + let txs: Vec<_> = select(&mempool).into_iter().map(|tx| (tx, None)).collect(); + let prevouts = txs + .iter() + .flat_map(|(tx, _)| &tx.input) + .filter(|txin| has_prevout(txin)) + .filter_map(|txin| { + mempool + .lookup_txo(&txin.previous_output) + .map(|txout| (txin.previous_output, txout)) + }) + .collect(); + Self { txs, prevouts } + } + + fn prepare(self, query: &Query, config: &Config) -> Result, HttpError> { + let mut prevouts = self.prevouts; + let confirmed_prevouts: BTreeSet<_> = self + .txs + .iter() + .filter(|(_, blockid)| blockid.is_some()) + .flat_map(|(tx, _)| tx.input.iter()) + .filter(|txin| has_prevout(txin)) + .map(|txin| txin.previous_output) + .collect(); + prepare_txs( + self.txs, + |mut outpoints| { + outpoints.retain(|outpoint| !prevouts.contains_key(outpoint)); + prevouts.extend( + query + .chain() + .lookup_txos(outpoints) + .map_err(|err| HttpError::mempool_prevout(err, query, &confirmed_prevouts))?, + ); + Ok(prevouts) + }, + config, + ) + } +} + +fn prepare_history( + query: &Query, + config: &Config, + mempool_history: impl FnOnce(&Mempool) -> Vec, + chain_history: impl FnOnce() -> errors::Result>, +) -> Result, HttpError> { + // Read mempool first: a transaction confirmed between these reads remains + // in at least one result. Prefer the confirmed entry when both contain it. + let mut snapshot = MempoolTxs::capture(query, mempool_history); + let chain_txs = chain_history().map_err(HttpError::lookup)?; + let confirmed: BTreeSet<_> = chain_txs.iter().map(|(tx, _)| tx.compute_txid()).collect(); + snapshot + .txs + .retain(|(tx, _)| !confirmed.contains(&tx.compute_txid())); + snapshot.txs.extend( + chain_txs + .into_iter() + .map(|(tx, blockid)| (tx, Some(blockid))), + ); + snapshot.prepare(query, config) +} + +fn prepare_tx( + hash: &Txid, query: &Query, config: &Config, -) -> Vec { +) -> Result<(TransactionValue, u32), HttpError> { + if let Some(tx) = query.chain().lookup_txn(hash, None).map_err(HttpError::lookup)? { + let blockid = query.chain().try_tx_confirming_block(hash).map_err(HttpError::lookup)?; + let ttl = ttl_by_depth(blockid.map(|b| b.height), query); + let tx = prepare_txs( + vec![(tx, blockid)], + |outpoints| query.chain().lookup_txos(outpoints).map_err(HttpError::lookup), + config, + )? + .remove(0); + return Ok((tx, ttl)); + } + + let snapshot = MempoolTxs::capture(query, |mempool| { + mempool.lookup_txn(hash).into_iter().collect() + }); + prepare_captured_tx(snapshot, hash, query, config) +} + +fn prepare_captured_tx( + mut snapshot: MempoolTxs, + hash: &Txid, + query: &Query, + config: &Config, +) -> Result<(TransactionValue, u32), HttpError> { + if snapshot.txs.is_empty() { + let tx = query + .chain() + .lookup_txn(hash, None) + .map_err(HttpError::lookup)? + .ok_or_else(|| HttpError::not_found("Transaction not found".to_string()))?; + snapshot.txs.push((tx, None)); + } + // Look up status after transaction selection, including the mempool path. + let blockid = query.chain().try_tx_confirming_block(hash).map_err(HttpError::lookup)?; + snapshot.txs[0].1 = blockid; + let ttl = ttl_by_depth(blockid.map(|b| b.height), query); + Ok((snapshot.prepare(query, config)?.remove(0), ttl)) +} + +fn prepare_txs( + txs: Vec<(Transaction, Option)>, + lookup_txos: F, + config: &Config, +) -> Result, HttpError> +where + F: FnOnce(BTreeSet) -> Result, HttpError>, +{ let outpoints = txs .iter() .flat_map(|(tx, _)| { @@ -502,11 +626,12 @@ fn prepare_txs( }) .collect(); - let prevouts = query.lookup_txos(outpoints); + let prevouts = lookup_txos(outpoints)?; - txs.into_iter() + Ok(txs + .into_iter() .map(|(tx, blockid)| TransactionValue::new(tx, blockid, &prevouts, config)) - .collect() + .collect()) } fn spawn_conn( @@ -807,7 +932,8 @@ fn handle_blocking_request( let hash = BlockHash::from_str(hash)?; let txids = query .chain() - .get_block_txids(&hash) + .try_get_block_txids(&hash) + .map_err(HttpError::lookup)? .ok_or_else(|| HttpError::not_found("Block not found".to_string()))?; json_response(txids, TTL_LONG) } @@ -840,7 +966,8 @@ fn handle_blocking_request( let index: usize = index.parse()?; let txids = query .chain() - .get_block_txids(&hash) + .try_get_block_txids(&hash) + .map_err(HttpError::lookup)? .ok_or_else(|| HttpError::not_found("Block not found".to_string()))?; if index >= txids.len() { bail!(HttpError::not_found("tx index out of range".to_string())); @@ -849,7 +976,7 @@ fn handle_blocking_request( } (&Method::GET, Some(&"block"), Some(hash), Some(&"txs"), start_index, None) => { let hash = BlockHash::from_str(hash)?; - + // Add lightweight validation that block exists before fetching transactions, // to avoid expensive lookups in case of invalid block hash query.chain().get_block_header(&hash) @@ -870,7 +997,8 @@ fn handle_blocking_request( let txs = query .chain() - .get_block_txs(&hash, start_index, CHAIN_TXS_PER_PAGE)? + .get_block_txs(&hash, start_index, CHAIN_TXS_PER_PAGE) + .map_err(HttpError::block_txs)? .into_iter() .map(|tx| (tx, blockid)) .collect(); @@ -878,12 +1006,19 @@ fn handle_blocking_request( // XXX stale blocks alway get TTL_SHORT let ttl = ttl_by_depth(blockid.map(|b| b.height), query); - json_response(prepare_txs(txs, query, config), ttl) + json_response( + prepare_txs( + txs, + |outpoints| query.chain().lookup_txos(outpoints).map_err(HttpError::lookup), + config, + )?, + ttl, + ) } (&Method::GET, Some(script_type @ &"address"), Some(script_str), None, None, None) | (&Method::GET, Some(script_type @ &"scripthash"), Some(script_str), None, None, None) => { let script_hash = to_scripthash(script_type, script_str, config.network_type)?; - let stats = query.stats(&script_hash[..]); + let stats = query.stats(&script_hash[..])?; json_response( json!({ *script_type: script_str, @@ -910,26 +1045,18 @@ fn handle_blocking_request( None, ) => { let script_hash = to_scripthash(script_type, script_str, config.network_type)?; - - let mut txs = vec![]; - - txs.extend( - query - .mempool() - .history(&script_hash[..], MAX_MEMPOOL_TXS) - .into_iter() - .map(|tx| (tx, None)), - ); - - txs.extend( - query - .chain() - .history(&script_hash[..], None, CHAIN_TXS_PER_PAGE) - .into_iter() - .map(|(tx, blockid)| (tx, Some(blockid))), - ); - - json_response(prepare_txs(txs, query, config), TTL_SHORT) + let txs = prepare_history( + query, + config, + |mempool| mempool.history(&script_hash[..], MAX_MEMPOOL_TXS), + || { + query + .chain() + .history(&script_hash[..], None, CHAIN_TXS_PER_PAGE) + }, + )?; + + json_response(txs, TTL_SHORT) } ( @@ -950,7 +1077,6 @@ fn handle_blocking_request( ) => { let script_hash = to_scripthash(script_type, script_str, config.network_type)?; let last_seen_txid = last_seen_txid.and_then(|txid| Txid::from_str(txid).ok()); - let txs = query .chain() .history( @@ -958,11 +1084,19 @@ fn handle_blocking_request( last_seen_txid.as_ref(), CHAIN_TXS_PER_PAGE, ) + .map_err(HttpError::lookup)? .into_iter() .map(|(tx, blockid)| (tx, Some(blockid))) .collect(); - json_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response( + prepare_txs( + txs, + |outpoints| query.chain().lookup_txos(outpoints).map_err(HttpError::lookup), + config, + )?, + TTL_SHORT, + ) } ( &Method::GET, @@ -981,15 +1115,12 @@ fn handle_blocking_request( None, ) => { let script_hash = to_scripthash(script_type, script_str, config.network_type)?; + let txs = MempoolTxs::capture(query, |mempool| { + mempool.history(&script_hash[..], MAX_MEMPOOL_TXS) + }) + .prepare(query, config)?; - let txs = query - .mempool() - .history(&script_hash[..], MAX_MEMPOOL_TXS) - .into_iter() - .map(|tx| (tx, None)) - .collect(); - - json_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(txs, TTL_SHORT) } ( @@ -1026,13 +1157,7 @@ fn handle_blocking_request( } (&Method::GET, Some(&"tx"), Some(hash), None, None, None) => { let hash = Txid::from_str(hash)?; - let tx = query - .lookup_txn(&hash) - .ok_or_else(|| HttpError::not_found("Transaction not found".to_string()))?; - let blockid = query.chain().tx_confirming_block(&hash); - let ttl = ttl_by_depth(blockid.as_ref().map(|b| b.height), query); - - let tx = prepare_txs(vec![(tx, blockid)], query, config).remove(0); + let (tx, ttl) = prepare_tx(&hash, query, config)?; json_response(tx, ttl) } @@ -1118,6 +1243,7 @@ fn handle_blocking_request( let hash = Txid::from_str(hash)?; let tx = query .lookup_txn(&hash) + .map_err(HttpError::lookup)? .ok_or_else(|| HttpError::not_found("Transaction not found".to_string()))?; let spends: Vec = query .lookup_tx_spends(&tx) @@ -1257,26 +1383,18 @@ fn handle_blocking_request( #[cfg(feature = "liquid")] (&Method::GET, Some(&"asset"), Some(asset_str), Some(&"txs"), None, None) => { let asset_id = AssetId::from_str(asset_str)?; - - let mut txs = vec![]; - - txs.extend( - query - .mempool() - .asset_history(&asset_id, MAX_MEMPOOL_TXS) - .into_iter() - .map(|tx| (tx, None)), - ); - - txs.extend( - query - .chain() - .asset_history(&asset_id, None, CHAIN_TXS_PER_PAGE) - .into_iter() - .map(|(tx, blockid)| (tx, Some(blockid))), - ); - - json_response(prepare_txs(txs, query, config), TTL_SHORT) + let txs = prepare_history( + query, + config, + |mempool| mempool.asset_history(&asset_id, MAX_MEMPOOL_TXS), + || { + query + .chain() + .asset_history(&asset_id, None, CHAIN_TXS_PER_PAGE) + }, + )?; + + json_response(txs, TTL_SHORT) } #[cfg(feature = "liquid")] @@ -1290,29 +1408,33 @@ fn handle_blocking_request( ) => { let asset_id = AssetId::from_str(asset_str)?; let last_seen_txid = last_seen_txid.and_then(|txid| Txid::from_str(txid).ok()); - let txs = query .chain() .asset_history(&asset_id, last_seen_txid.as_ref(), CHAIN_TXS_PER_PAGE) + .map_err(HttpError::lookup)? .into_iter() .map(|(tx, blockid)| (tx, Some(blockid))) .collect(); - json_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response( + prepare_txs( + txs, + |outpoints| query.chain().lookup_txos(outpoints).map_err(HttpError::lookup), + config, + )?, + TTL_SHORT, + ) } #[cfg(feature = "liquid")] (&Method::GET, Some(&"asset"), Some(asset_str), Some(&"txs"), Some(&"mempool"), None) => { let asset_id = AssetId::from_str(asset_str)?; + let txs = MempoolTxs::capture(query, |mempool| { + mempool.asset_history(&asset_id, MAX_MEMPOOL_TXS) + }) + .prepare(query, config)?; - let txs = query - .mempool() - .asset_history(&asset_id, MAX_MEMPOOL_TXS) - .into_iter() - .map(|tx| (tx, None)) - .collect(); - - json_response(prepare_txs(txs, query, config), TTL_SHORT) + json_response(txs, TTL_SHORT) } #[cfg(feature = "liquid")] @@ -1527,6 +1649,55 @@ impl HttpError { fn forbidden(msg: String) -> Self { HttpError(StatusCode::FORBIDDEN, msg) } + + fn lookup(err: errors::Error) -> Self { + // A history that exceeds the configured scan limit is a property of the + // requested scripthash, not a transient server-side failure. + if let errors::ErrorKind::TooBigHistory = err.kind() { + return HttpError::from(err.to_string()); + } + let status = StatusCode::SERVICE_UNAVAILABLE; + warn!("REST lookup failed status='{}' err='{:?}'", status, err); + HttpError(status, err.to_string()) + } + + fn block_txs(err: errors::Error) -> Self { + match err.kind() { + errors::ErrorKind::InvalidParams(_) => HttpError::from(err.to_string()), + _ => HttpError::lookup(err), + } + } + + fn mempool_prevout( + err: errors::Error, + query: &Query, + confirmed_prevouts: &BTreeSet, + ) -> Self { + if let errors::ErrorKind::MissingTxo(missing) = err.kind() { + let used_by_confirmed_tx = confirmed_prevouts + .iter() + .any(|outpoint| outpoint.to_string() == *missing); + let confirmed_parent = missing_txo_txid(missing) + .map(|txid| query.chain().try_tx_confirming_block(&txid)); + let confirmed_parent = match confirmed_parent { + Some(Ok(blockid)) => blockid.is_some(), + Some(Err(err)) => return HttpError::lookup(err), + None => false, + }; + if !used_by_confirmed_tx && !confirmed_parent { + warn!("REST mempool prevout vanished err='{:?}'", err); + return HttpError::not_found(err.to_string()); + } + } + HttpError::lookup(err) + } +} + +/// Parse the txid out of a `MissingTxo` outpoint string. Elements prefixes its outpoint +/// display with `[elements]`. +fn missing_txo_txid(missing: &str) -> Option { + let (txid, _) = missing.rsplit_once(':')?; + Txid::from_str(txid.strip_prefix("[elements]").unwrap_or(txid)).ok() } impl From for HttpError { @@ -1621,13 +1792,22 @@ impl From for HttpError { #[cfg(test)] mod tests { - use crate::rest::{is_block_template_request, HttpError}; + use crate::rest::{ + is_block_template_request, ttl_by_confirmations, HttpError, TTL_LONG, TTL_SHORT, + }; use crate::{errors, errors::ErrorKind}; use http_body_util::BodyExt; use hyper::{Method, StatusCode}; use serde_json::Value; use std::collections::HashMap; + #[test] + fn ttl_by_confirmations_survives_a_reorg_race() { + assert_eq!(ttl_by_confirmations(110, 100), TTL_LONG); + assert_eq!(ttl_by_confirmations(105, 100), TTL_SHORT); + assert_eq!(ttl_by_confirmations(99, 100), TTL_SHORT); + } + #[test] fn block_template_is_the_only_route_kept_on_the_async_runtime() { let is_async = |method: Method, uri: &str| { @@ -1677,6 +1857,29 @@ mod tests { assert_eq!(rejected.0, StatusCode::BAD_REQUEST); } + #[test] + fn storage_lookup_failures_are_unavailable() { + let missing = HttpError::lookup(errors::Error::from(ErrorKind::MissingTxo( + "abc:0".to_string(), + ))); + assert_eq!(missing.0, StatusCode::SERVICE_UNAVAILABLE); + + let corrupt = HttpError::lookup(errors::Error::from("failed to parse TxOut")); + assert_eq!(corrupt.0, StatusCode::SERVICE_UNAVAILABLE); + } + + #[test] + fn block_page_errors_distinguish_invalid_index_from_storage_failure() { + let invalid = HttpError::block_txs(errors::Error::from(ErrorKind::InvalidParams( + "start index out of range".to_string(), + ))); + assert_eq!(invalid.0, StatusCode::BAD_REQUEST); + assert_eq!(invalid.1, "start index out of range"); + + let storage = HttpError::block_txs(errors::Error::from("missing indexed transaction")); + assert_eq!(storage.0, StatusCode::SERVICE_UNAVAILABLE); + } + #[test] fn test_parse_query_param() { let mut query_params = HashMap::new(); @@ -1827,3 +2030,7 @@ mod tests { } } } + +#[cfg(test)] +#[path = "rest/mempool_tests.rs"] +mod mempool_tests; diff --git a/src/rest/mempool_tests.rs b/src/rest/mempool_tests.rs new file mode 100644 index 000000000..b6d3eb6c9 --- /dev/null +++ b/src/rest/mempool_tests.rs @@ -0,0 +1,274 @@ +//! Deterministic races through the private preparation functions used by REST. +use super::*; +use crate::test_common as common; +use crate::test_common::Result; + +#[test] +#[cfg(not(feature = "liquid"))] +fn prevout_lookup_survives_ancestor_eviction() -> Result<()> { + let mut tester = common::TestRunner::new()?; + + // Parent: a wallet spend, left unconfirmed. + let addr_parent = tester.newaddress()?; + let parent_txid = tester.send(&addr_parent, "1.0 BTC".parse().unwrap())?; + + let parent_tx = tester.get_raw_transaction(parent_txid)?; + let parent_spk = addr_parent.script_pubkey(); + let (parent_vout, parent_txout) = parent_tx + .output + .iter() + .enumerate() + .find(|(_, o)| o.script_pubkey == parent_spk) + .expect("funding output not found in parent tx"); + + // Child: explicitly spends the parent's own unconfirmed output, so the + // ancestor relationship (and thus the prevout dependency) is guaranteed. + let addr_child = tester.newaddress()?; + let child_value = parent_txout.value - bitcoin::Amount::from_sat(1000); + let raw_child: String = tester.node_client().call( + "createrawtransaction", + &[ + json!([{ "txid": parent_txid.to_string(), "vout": parent_vout }]), + json!({ addr_child.to_string(): child_value.to_btc() }), + ], + )?; + let signed: serde_json::Value = tester + .node_client() + .call("signrawtransactionwithwallet", &[json!(raw_child)])?; + assert_eq!(signed["complete"].as_bool(), Some(true)); + let child_txid_str: String = tester + .node_client() + .call("sendrawtransaction", &[signed["hex"].clone()])?; + let child_txid = child_txid_str.parse().unwrap(); + + tester.sync()?; + + let query = Arc::clone(tester.query()); + let config = Arc::clone(tester.config()); + let snapshot = MempoolTxs::capture(&query, |mempool| { + assert!(tester.mempool().try_write().is_err()); + vec![mempool.lookup_txn(&child_txid).unwrap()] + }); + let child_tx = snapshot.txs[0].0.clone(); + let parent_outpoint = OutPoint::new(parent_txid, parent_vout as u32); + assert!(snapshot.prevouts.contains_key(&parent_outpoint)); + assert!(tester.mempool().try_write().is_ok()); + + // EVICT: RBF-replace the parent on the node by directly double-spending + // its inputs (not via `bumpfee`, which refuses once the wallet sees a + // real descendant -- exactly the descendant we just created). This also + // invalidates and evicts the child, since its input no longer exists. + // Then apply that eviction to electrs's local mempool view -- exactly + // what the background sync loop does on its next tick, just called + // directly and synchronously here instead of racing its timer. + let gettx: serde_json::Value = tester + .node_client() + .call("gettransaction", &[json!(parent_txid.to_string())])?; + let parent_fee = bitcoin::Amount::from_btc(gettx["fee"].as_f64().unwrap().abs()).unwrap(); + let total_output: bitcoin::Amount = parent_tx.output.iter().map(|o| o.value).sum(); + let total_input = total_output + parent_fee; + let higher_fee = bitcoin::Amount::from_sat(50_000); // well above the parent+child's combined fee + let replacement_value = total_input - higher_fee; + + let addr_conflict = tester.newaddress()?; + let raw_conflict: String = tester.node_client().call( + "createrawtransaction", + &[ + json!(parent_tx + .input + .iter() + .map(|txin| json!({ + "txid": txin.previous_output.txid.to_string(), + "vout": txin.previous_output.vout, + })) + .collect::>()), + json!({ addr_conflict.to_string(): replacement_value.to_btc() }), + ], + )?; + let signed_conflict: serde_json::Value = tester + .node_client() + .call("signrawtransactionwithwallet", &[json!(raw_conflict)])?; + assert_eq!(signed_conflict["complete"].as_bool(), Some(true)); + let _replacement_txid: String = tester + .node_client() + .call("sendrawtransaction", &[signed_conflict["hex"].clone()])?; + + let tip = tester.get_best_block_hash()?; + assert_eq!( + Mempool::update(&tester.mempool(), &tester.daemon(), &tip)?, + crate::new_index::MempoolSyncStatus::Synced + ); + + // Exercise the actual REST preparation path after the ancestor is gone. + // Keep a writer guard during preparation: no mempool re-read is allowed. + let writer = tester.mempool().try_write().unwrap(); + let prepared = snapshot.prepare(&query, &config).unwrap(); + assert_eq!(prepared[0].txid, child_txid); + assert_eq!(prepared[0].fee, 1000); + let body = serde_json::to_value(&prepared[0])?; + assert_eq!( + body["vin"][0]["prevout"]["value"], + parent_txout.value.to_sat() + ); + + // Losing the captured prevout must reach REST's 404 mapping, not abort. + let stale = MempoolTxs { + txs: vec![(child_tx, None)], + prevouts: HashMap::new(), + }; + let error = stale.prepare(&query, &config).err().unwrap(); + assert_eq!(error.0, StatusCode::NOT_FOUND); + drop(writer); + Ok(()) +} + +#[test] +fn history_keeps_transaction_confirmed_between_reads() -> Result<()> { + for confirm_before_chain_read in [false, true] { + let mut tester = common::TestRunner::new()?; + let address = tester.newaddress()?; + let txid = tester.send(&address, "1 BTC".parse().unwrap())?; + let script_hash = compute_script_hash(&address.script_pubkey()); + let query = Arc::clone(tester.query()); + let config = Arc::clone(tester.config()); + let txs = prepare_history( + &query, + &config, + |mempool| mempool.history(&script_hash, MAX_MEMPOOL_TXS), + || { + assert!(tester.mempool().try_write().is_ok()); + if confirm_before_chain_read { + tester.mine().unwrap(); + } + let txs = query + .chain() + .history(&script_hash, None, CHAIN_TXS_PER_PAGE); + if !confirm_before_chain_read { + // A chain-first implementation loses this transaction when + // it subsequently reads the now-empty mempool. + tester.mine().unwrap(); + } + txs + }, + ) + .unwrap(); + let matches: Vec<_> = txs.iter().filter(|tx| tx.txid == txid).collect(); + assert_eq!(matches.len(), 1); + assert_eq!( + matches[0].status.as_ref().unwrap().confirmed, + confirm_before_chain_read + ); + } + Ok(()) +} + +#[test] +fn selected_mempool_transaction_uses_current_confirmation_status() -> Result<()> { + let mut tester = common::TestRunner::new()?; + let address = tester.newaddress()?; + let txid = tester.send(&address, "1 BTC".parse().unwrap())?; + let query = Arc::clone(tester.query()); + let config = Arc::clone(tester.config()); + let snapshot = MempoolTxs::capture(&query, |mempool| vec![mempool.lookup_txn(&txid).unwrap()]); + let blockhash = tester.mine()?; + let writer = tester.mempool().try_write().unwrap(); + let (tx, ttl) = prepare_captured_tx(snapshot, &txid, &query, &config).unwrap(); + let status = tx.status.unwrap(); + assert!(status.confirmed); + assert_eq!(status.block_hash, Some(blockhash)); + assert_eq!(ttl, ttl_by_depth(status.block_height, &query)); + drop(writer); + + // The same path must also find a transaction already removed from mempool. + let snapshot = MempoolTxs::capture(&query, |mempool| { + mempool.lookup_txn(&txid).into_iter().collect() + }); + assert!(snapshot.txs.is_empty()); + let (tx, _) = prepare_captured_tx(snapshot, &txid, &query, &config).unwrap(); + assert_eq!(tx.status.unwrap().block_hash, Some(blockhash)); + Ok(()) +} + +#[test] +fn confirmed_transaction_does_not_wait_for_mempool_writer() -> Result<()> { + use std::sync::mpsc; + use std::time::Duration; + + let mut tester = common::TestRunner::new()?; + let address = tester.newaddress()?; + let txid = tester.send(&address, "1 BTC".parse().unwrap())?; + tester.mine()?; + + let query = Arc::clone(tester.query()); + let config = Arc::clone(tester.config()); + let writer = tester.mempool().try_write().unwrap(); + let (sender, receiver) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let result = prepare_tx(&txid, &query, &config).map(|(tx, _)| tx.status.unwrap().confirmed); + sender.send(result).unwrap(); + }); + + let completed = receiver.recv_timeout(Duration::from_secs(5)); + drop(writer); + worker.join().unwrap(); + assert!(completed.unwrap().unwrap()); + Ok(()) +} + +#[test] +fn missing_confirmed_parent_is_unavailable() -> Result<()> { + let mut tester = common::TestRunner::new()?; + let address = tester.newaddress()?; + let parent = tester.send(&address, "1 BTC".parse().unwrap())?; + tester.mine()?; + + let missing = errors::Error::from(errors::ErrorKind::MissingTxo( + OutPoint::new(parent, 0).to_string(), + )); + let status = HttpError::mempool_prevout(missing, tester.query(), &BTreeSet::new()).0; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + Ok(()) +} + +#[test] +#[cfg(feature = "liquid")] +fn asset_history_keeps_transaction_confirmed_between_reads() -> Result<()> { + use elementsd::bitcoincore_rpc::RpcApi; + for confirm_before_chain_read in [false, true] { + let mut tester = common::TestRunner::new()?; + let issued: serde_json::Value = tester + .node_client() + .call("issueasset", &[json!(1), json!(0)])?; + let txid: Txid = issued["txid"].as_str().unwrap().parse().unwrap(); + let asset: AssetId = issued["asset"].as_str().unwrap().parse().unwrap(); + tester.sync()?; + let query = Arc::clone(tester.query()); + let config = Arc::clone(tester.config()); + let txs = prepare_history( + &query, + &config, + |mempool| mempool.asset_history(&asset, MAX_MEMPOOL_TXS), + || { + assert!(tester.mempool().try_write().is_ok()); + if confirm_before_chain_read { + tester.mine().unwrap(); + } + let txs = query + .chain() + .asset_history(&asset, None, CHAIN_TXS_PER_PAGE); + if !confirm_before_chain_read { + tester.mine().unwrap(); + } + txs + }, + ) + .unwrap(); + let matches: Vec<_> = txs.iter().filter(|tx| tx.txid == txid).collect(); + assert_eq!(matches.len(), 1); + assert_eq!( + matches[0].status.as_ref().unwrap().confirmed, + confirm_before_chain_read + ); + } + Ok(()) +} diff --git a/src/util/electrum_merkle.rs b/src/util/electrum_merkle.rs index 7bf813afd..bce56dc43 100644 --- a/src/util/electrum_merkle.rs +++ b/src/util/electrum_merkle.rs @@ -2,40 +2,367 @@ use crate::chain::{BlockHash, Txid}; use crate::errors::*; use crate::new_index::ChainQuery; use bitcoin::hashes::{sha256d::Hash as Sha256dHash, Hash}; +use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use electrs_macros::trace; -static INFLIGHT_CHECKPOINT_PROOFS: AtomicUsize = AtomicUsize::new(0); +const MAX_CACHE_ENTRIES: usize = 256; -struct InflightCheckpointProofGuard; +struct CachedLevels { + cp_hash: BlockHash, + levels: Vec>, + size_bytes: usize, +} + +impl CachedLevels { + fn new(cp_hash: BlockHash, levels: Vec>) -> Self { + let size_bytes = levels.iter().map(|level| level.len()).sum::() + * std::mem::size_of::(); + Self { + cp_hash, + levels, + size_bytes, + } + } +} -impl InflightCheckpointProofGuard { - fn acquire(limit: usize) -> Result { - let mut current = INFLIGHT_CHECKPOINT_PROOFS.load(Ordering::Relaxed); +struct RebuildPermit<'a> { + counter: &'a AtomicUsize, +} + +impl<'a> RebuildPermit<'a> { + fn acquire(counter: &'a AtomicUsize, limit: usize) -> Result { + let mut current = counter.load(Ordering::Relaxed); loop { if current >= limit { - bail!("too many concurrent checkpoint merkle proof requests, try again later"); + bail!("too many concurrent checkpoint merkle proof rebuilds, try again later"); } - match INFLIGHT_CHECKPOINT_PROOFS.compare_exchange_weak( + match counter.compare_exchange_weak( current, current + 1, Ordering::AcqRel, Ordering::Relaxed, ) { - Ok(_) => return Ok(Self), + Ok(_) => return Ok(Self { counter }), + Err(observed) => current = observed, + } + } + } +} + +impl<'a> Drop for RebuildPermit<'a> { + fn drop(&mut self) { + self.counter.fetch_sub(1, Ordering::AcqRel); + } +} + +struct MemoryPermit<'a> { + counter: &'a AtomicUsize, + weight_bytes: usize, +} + +impl<'a> MemoryPermit<'a> { + fn acquire(counter: &'a AtomicUsize, limit_bytes: usize, weight_bytes: usize) -> Result { + let mut current = counter.load(Ordering::Relaxed); + loop { + if current > 0 && current.saturating_add(weight_bytes) > limit_bytes { + bail!("checkpoint merkle proof rebuild would exceed the configured memory budget, try again later"); + } + match counter.compare_exchange_weak( + current, + current + weight_bytes, + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => { + return Ok(Self { + counter, + weight_bytes, + }) + } Err(observed) => current = observed, } } } } -impl Drop for InflightCheckpointProofGuard { +impl<'a> Drop for MemoryPermit<'a> { fn drop(&mut self) { - INFLIGHT_CHECKPOINT_PROOFS.fetch_sub(1, Ordering::AcqRel); + self.counter.fetch_sub(self.weight_bytes, Ordering::AcqRel); } } +fn estimated_peak_build_bytes(cp_height: usize) -> usize { + (cp_height + 1) + .saturating_mul(std::mem::size_of::()) + .saturating_mul(3) +} + +struct MemoryCharge { + counter: Arc, + bytes: usize, +} + +impl MemoryCharge { + fn new(counter: Arc, bytes: usize) -> Self { + counter.fetch_add(bytes, Ordering::AcqRel); + Self { counter, bytes } + } +} + +impl Drop for MemoryCharge { + fn drop(&mut self) { + self.counter.fetch_sub(self.bytes, Ordering::AcqRel); + } +} + +struct RetainedTree { + tree: Arc, + _charge: MemoryCharge, +} + +struct BuildSlot { + waiters: AtomicUsize, + last_built: Mutex>, +} + +struct BuildSlotGuard<'a> { + cache: &'a CheckpointMerkleCache, + cp_height: usize, + slot: Arc, +} + +impl<'a> Drop for BuildSlotGuard<'a> { + fn drop(&mut self) { + let mut locks = self.cache.build_locks.lock().unwrap(); + if self.slot.waiters.fetch_sub(1, Ordering::AcqRel) == 1 { + if let Some(existing) = locks.get(&self.cp_height) { + if Arc::ptr_eq(existing, &self.slot) { + locks.remove(&self.cp_height); + } + } + } + } +} + +pub struct CheckpointMerkleCache { + entries: Mutex)>>, + build_locks: Mutex>>, + inflight_rebuilds: AtomicUsize, + inflight_build_bytes: Arc, + capacity_bytes: usize, + cache_capacity_bytes: usize, +} + +impl CheckpointMerkleCache { + /// Cached trees may use half of `capacity_bytes`, so a full cache always leaves at least + /// the other half for rebuild working memory. + pub fn new(capacity_bytes: usize) -> Self { + Self::with_cache_capacity(capacity_bytes, capacity_bytes / 2) + } + + fn with_cache_capacity(capacity_bytes: usize, cache_capacity_bytes: usize) -> Self { + Self { + entries: Mutex::new(Vec::new()), + build_locks: Mutex::new(HashMap::new()), + inflight_rebuilds: AtomicUsize::new(0), + inflight_build_bytes: Arc::new(AtomicUsize::new(0)), + capacity_bytes, + cache_capacity_bytes: cache_capacity_bytes.min(capacity_bytes), + } + } + + fn get_valid(&self, cp_height: usize, cp_hash: BlockHash) -> Option> { + let entries = self.entries.lock().unwrap(); + entries.iter().find_map(|(h, cached)| { + (*h == cp_height && cached.cp_hash == cp_hash).then(|| cached.clone()) + }) + } + + fn insert(&self, cp_height: usize, best_height: usize, cached: Arc) { + if cached.size_bytes > self.cache_capacity_bytes { + return; + } + + let mut entries = self.entries.lock().unwrap(); + entries.retain(|(h, _)| *h != cp_height && *h <= best_height); + if cp_height > best_height { + return; + } + + let mut total_bytes: usize = entries.iter().map(|(_, c)| c.size_bytes).sum(); + let mut remaining_entries = entries.len(); + let mut eviction_positions: Vec = entries + .iter() + .enumerate() + .filter_map(|(pos, (height, _))| (*height < cp_height).then_some(pos)) + .collect(); + eviction_positions.sort_unstable_by_key(|pos| entries[*pos].0); + + let mut evictions_needed = 0; + while total_bytes + cached.size_bytes > self.cache_capacity_bytes + || remaining_entries >= MAX_CACHE_ENTRIES + { + let Some(pos) = eviction_positions.get(evictions_needed).copied() else { + return; + }; + total_bytes -= entries[pos].1.size_bytes; + remaining_entries -= 1; + evictions_needed += 1; + } + + let mut selected_positions = eviction_positions[..evictions_needed].to_vec(); + selected_positions.sort_unstable_by(|a, b| b.cmp(a)); + for pos in selected_positions { + entries.remove(pos); + } + + entries.push((cp_height, cached)); + } + + fn build_lock_for(&self, cp_height: usize) -> BuildSlotGuard<'_> { + let mut locks = self.build_locks.lock().unwrap(); + let slot = locks + .entry(cp_height) + .or_insert_with(|| { + Arc::new(BuildSlot { + waiters: AtomicUsize::new(0), + last_built: Mutex::new(None), + }) + }) + .clone(); + slot.waiters.fetch_add(1, Ordering::AcqRel); + BuildSlotGuard { + cache: self, + cp_height, + slot, + } + } + + fn get_or_build( + &self, + cp_height: usize, + current_best_height: impl FnOnce() -> usize, + concurrency_limit: usize, + snapshot_cp_hash_and_sibling: impl Fn() -> Result<(BlockHash, Sha256dHash)>, + build: impl FnOnce() -> Result<(BlockHash, Vec>, Sha256dHash)>, + ) -> Result<(Arc, Sha256dHash)> { + let (cp_hash, sibling) = snapshot_cp_hash_and_sibling()?; + if let Some(cached) = self.get_valid(cp_height, cp_hash) { + return Ok((cached, sibling)); + } + + let build_lock = self.build_lock_for(cp_height); + self.build_under_lock( + cp_height, + current_best_height, + cp_hash, + sibling, + concurrency_limit, + &build_lock.slot, + build, + ) + } + + fn build_under_lock( + &self, + cp_height: usize, + current_best_height: impl FnOnce() -> usize, + cp_hash: BlockHash, + sibling: Sha256dHash, + concurrency_limit: usize, + build_lock: &BuildSlot, + build: impl FnOnce() -> Result<(BlockHash, Vec>, Sha256dHash)>, + ) -> Result<(Arc, Sha256dHash)> { + let mut last_built = build_lock + .last_built + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + + if let Some(cached) = self.get_valid(cp_height, cp_hash) { + return Ok((cached, sibling)); + } + if let Some(retained) = last_built.as_ref().filter(|r| r.tree.cp_hash == cp_hash) { + return Ok((retained.tree.clone(), sibling)); + } + + let _permit = RebuildPermit::acquire(&self.inflight_rebuilds, concurrency_limit)?; + let entries = self.entries.lock().unwrap(); + let resident_bytes: usize = entries.iter().map(|(_, cached)| cached.size_bytes).sum(); + let _memory_permit = MemoryPermit::acquire( + &self.inflight_build_bytes, + self.capacity_bytes.saturating_sub(resident_bytes), + estimated_peak_build_bytes(cp_height), + )?; + drop(entries); + let (cp_hash, levels, sibling) = build()?; + let cached = Arc::new(CachedLevels::new(cp_hash, levels)); + self.insert(cp_height, current_best_height(), cached.clone()); + *last_built = Some(RetainedTree { + _charge: MemoryCharge::new(self.inflight_build_bytes.clone(), cached.size_bytes), + tree: cached.clone(), + }); + Ok((cached, sibling)) + } +} + +fn merklize(left: Sha256dHash, right: Sha256dHash) -> Sha256dHash { + let mut data = [0u8; 64]; + data[..32].copy_from_slice(&left[..]); + data[32..].copy_from_slice(&right[..]); + Sha256dHash::hash(&data) +} + +fn build_levels_above_leaves(leaves: &[Sha256dHash]) -> Result>> { + ensure!(!leaves.is_empty(), "missing block headers"); + let mut current: Vec = leaves.to_vec(); + + let mut levels = Vec::new(); + while current.len() > 1 { + if current.len() % 2 != 0 { + let last = *current.last().unwrap(); + current.reserve_exact(1); + current.push(last); + } + current = current + .chunks(2) + .map(|pair| merklize(pair[0], pair[1])) + .collect(); + levels.push(current.clone()); + } + Ok(levels) +} + +fn extract_branch_and_root( + levels_above_leaves: &[Vec], + sibling_leaf: Sha256dHash, + mut index: usize, +) -> Result<(Vec, Sha256dHash)> { + let mut merkle = vec![sibling_leaf]; + index /= 2; + + for level in &levels_above_leaves[..levels_above_leaves.len().saturating_sub(1)] { + let len = level.len(); + let sibling_index = if index % 2 == 0 { index + 1 } else { index - 1 }; + let sibling = if sibling_index < len { + level[sibling_index] + } else { + level[len - 1] + }; + merkle.push(sibling); + index /= 2; + } + + let root = *levels_above_leaves + .last() + .chain_err(|| "empty checkpoint merkle tree")? + .last() + .unwrap(); + Ok((merkle, root)) +} + #[trace] pub fn get_tx_merkle_proof( chain: &ChainQuery, @@ -61,7 +388,7 @@ pub fn get_header_merkle_proof( height: usize, cp_height: usize, checkpoint_proof_concurrency_limit: usize, -) -> Result<(Vec, Sha256dHash)> { +) -> Result<(Vec, Sha256dHash, BlockHash)> { if cp_height < height { bail!("cp_height #{} < height #{}", cp_height, height); } @@ -75,18 +402,71 @@ pub fn get_header_merkle_proof( ); } - let _guard = InflightCheckpointProofGuard::acquire(checkpoint_proof_concurrency_limit)?; + if cp_height == 0 { + let headers = chain.store().headers(); + let cp_hash = *headers + .header_by_height(0) + .chain_err(|| "missing block header at height 0")? + .hash(); + let root = Sha256dHash::from(cp_hash); + return Ok((vec![], root, cp_hash)); + } + + let sibling_index = if height % 2 == 0 { + height + 1 + } else { + height - 1 + }; + let sibling_height = sibling_index.min(cp_height); + + let snapshot_cp_hash_and_sibling = || -> Result<(BlockHash, Sha256dHash)> { + let headers = chain.store().headers(); + let cp_hash = *headers + .header_by_height(cp_height) + .chain_err(|| format!("missing block header at height {}", cp_height))? + .hash(); + let sibling = Sha256dHash::from( + *headers + .header_by_height(sibling_height) + .chain_err(|| format!("missing block header at height {}", sibling_height))? + .hash(), + ); + Ok((cp_hash, sibling)) + }; + + let build = || -> Result<(BlockHash, Vec>, Sha256dHash)> { + let headers = chain.store().headers(); + let cp_hash = *headers + .header_by_height(cp_height) + .chain_err(|| format!("missing block header at height {}", cp_height))? + .hash(); + let leaves: Vec = (0..=cp_height) + .map(|h| { + headers + .header_by_height(h) + .map(|entry| Sha256dHash::from(*entry.hash())) + }) + .collect::>>() + .chain_err(|| "missing block headers")?; + drop(headers); + + let sibling = leaves[sibling_height]; + let levels = build_levels_above_leaves(&leaves)?; + Ok((cp_hash, levels, sibling)) + }; - let heights: Vec = (0..=cp_height).collect(); - let header_hashes: Vec = heights - .into_iter() - .map(|height| chain.hash_by_height(height)) - .collect::>>() - .chain_err(|| "missing block headers")?; + let (cached, sibling) = chain.checkpoint_merkle_cache().get_or_build( + cp_height, + || chain.best_height(), + checkpoint_proof_concurrency_limit, + snapshot_cp_hash_and_sibling, + build, + )?; - let header_hashes = header_hashes.into_iter().map(Sha256dHash::from).collect(); - Ok(create_merkle_branch_and_root(header_hashes, height)) + let (branch, root) = extract_branch_and_root(&cached.levels, sibling, height)?; + Ok((branch, root, cached.cp_hash)) } + #[trace] pub fn get_id_from_pos( chain: &ChainQuery, @@ -116,11 +496,6 @@ pub fn get_id_from_pos( Ok((txid, branch)) } -fn merklize(left: Sha256dHash, right: Sha256dHash) -> Sha256dHash { - let data = [&left[..], &right[..]].concat(); - Sha256dHash::hash(&data) -} - fn create_merkle_branch_and_root( mut hashes: Vec, mut index: usize, @@ -145,30 +520,1056 @@ fn create_merkle_branch_and_root( #[cfg(test)] mod tests { use super::*; + use std::sync::Barrier; + use std::thread; + + const GENEROUS_TEST_CACHE_BYTES: usize = 1024 * 1024; + + fn full_share_cache(capacity_bytes: usize) -> CheckpointMerkleCache { + CheckpointMerkleCache::with_cache_capacity(capacity_bytes, capacity_bytes) + } + + fn leaves(n: usize) -> Vec { + (0..n).map(|i| Sha256dHash::hash(&[i as u8])).collect() + } + + fn sibling_leaf(leaves: &[Sha256dHash], cp_height: usize, height: usize) -> Sha256dHash { + let sibling_index = if height % 2 == 0 { + height + 1 + } else { + height - 1 + }; + leaves[sibling_index.min(cp_height)] + } + + #[test] + fn incremental_levels_match_naive_algorithm_for_every_height() { + for n in 1..=17usize { + let leaves = leaves(n); + let cp_height = n - 1; + + for height in 0..=cp_height { + let (expected_branch, expected_root) = + create_merkle_branch_and_root(leaves.clone(), height); + + let (branch, root) = if cp_height == 0 { + (vec![], leaves[0]) + } else { + let levels = build_levels_above_leaves(&leaves).unwrap(); + let sibling = sibling_leaf(&leaves, cp_height, height); + extract_branch_and_root(&levels, sibling, height).unwrap() + }; + + assert_eq!( + branch, expected_branch, + "branch mismatch for n={} height={}", + n, height + ); + assert_eq!( + root, expected_root, + "root mismatch for n={} height={}", + n, height + ); + } + } + } + + #[test] + fn cache_hit_serves_different_heights_without_rebuilding() { + let leaves = leaves(9); + let cp_height = 8; + let cp_hash = BlockHash::hash(&[7u8]); + + let cache = full_share_cache(GENEROUS_TEST_CACHE_BYTES); + let build_count = AtomicUsize::new(0); + + for height in [0usize, 3, 8] { + let sibling = sibling_leaf(&leaves, cp_height, height); + let (cached, _) = cache + .get_or_build( + cp_height, + || usize::MAX, + 1, + || Ok((cp_hash, sibling)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)) + }, + ) + .unwrap(); + let (branch, root) = extract_branch_and_root(&cached.levels, sibling, height).unwrap(); + let (expected_branch, expected_root) = + create_merkle_branch_and_root(leaves.clone(), height); + assert_eq!(branch, expected_branch); + assert_eq!(root, expected_root); + } + + assert_eq!( + build_count.load(Ordering::SeqCst), + 1, + "second and third lookups should hit the cache, not rebuild" + ); + } + + #[test] + fn stale_entry_is_rebuilt_after_checkpoint_hash_changes() { + let leaves = leaves(5); + let cp_height = 4; + let sibling = sibling_leaf(&leaves, cp_height, 0); + let old_hash = BlockHash::hash(&[1u8]); + let new_hash = BlockHash::hash(&[2u8]); + + let cache = full_share_cache(GENEROUS_TEST_CACHE_BYTES); + let build_count = AtomicUsize::new(0); + + cache + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((old_hash, sibling)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((old_hash, build_levels_above_leaves(&leaves)?, sibling)) + }, + ) + .unwrap(); + cache + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((old_hash, sibling)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((old_hash, build_levels_above_leaves(&leaves)?, sibling)) + }, + ) + .unwrap(); + assert_eq!(build_count.load(Ordering::SeqCst), 1); + + cache + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((new_hash, sibling)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((new_hash, build_levels_above_leaves(&leaves)?, sibling)) + }, + ) + .unwrap(); + assert_eq!( + build_count.load(Ordering::SeqCst), + 2, + "a changed checkpoint hash must trigger a rebuild" + ); + } + + #[test] + fn stale_entry_returns_correct_branch_and_root_after_checkpoint_hash_changes() { + let old_leaves = leaves(5); + let new_leaves: Vec = + (100..105).map(|i| Sha256dHash::hash(&[i as u8])).collect(); + let cp_height = 4; + let height = 1; + let old_sibling = sibling_leaf(&old_leaves, cp_height, height); + let new_sibling = sibling_leaf(&new_leaves, cp_height, height); + let old_hash = BlockHash::hash(&[1u8]); + let new_hash = BlockHash::hash(&[2u8]); + + let cache = full_share_cache(GENEROUS_TEST_CACHE_BYTES); + + let (old_cached, old_ret_sibling) = cache + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((old_hash, old_sibling)), + || { + Ok(( + old_hash, + build_levels_above_leaves(&old_leaves)?, + old_sibling, + )) + }, + ) + .unwrap(); + let (old_branch, old_root) = + extract_branch_and_root(&old_cached.levels, old_ret_sibling, height).unwrap(); + let (expected_old_branch, expected_old_root) = + create_merkle_branch_and_root(old_leaves.clone(), height); + assert_eq!(old_branch, expected_old_branch); + assert_eq!(old_root, expected_old_root); + + let (new_cached, new_ret_sibling) = cache + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((new_hash, new_sibling)), + || { + Ok(( + new_hash, + build_levels_above_leaves(&new_leaves)?, + new_sibling, + )) + }, + ) + .unwrap(); + let (new_branch, new_root) = + extract_branch_and_root(&new_cached.levels, new_ret_sibling, height).unwrap(); + let (expected_new_branch, expected_new_root) = + create_merkle_branch_and_root(new_leaves.clone(), height); + assert_eq!(new_branch, expected_new_branch); + assert_eq!(new_root, expected_new_root); + assert_ne!( + new_root, old_root, + "a rebuild after a checkpoint hash change must reflect the new chain state, not the stale cached one" + ); + } + + #[test] + fn insert_evicts_lowest_height_for_a_higher_value() { + let entry_leaves = leaves(2); + let entry_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() + * std::mem::size_of::(); + let cache = full_share_cache(5 * entry_bytes); + let leaf = entry_leaves[0]; + + for cp_height in [10usize, 20, 30, 40, 50] { + let hash = BlockHash::hash(&[cp_height as u8]); + cache + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((hash, leaf)), + || Ok((hash, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) + .unwrap(); + } + assert_eq!(cache.entries.lock().unwrap().len(), 5); + + let hash60 = BlockHash::hash(&[60u8]); + cache + .get_or_build( + 60, + || usize::MAX, + 4, + || Ok((hash60, leaf)), + || Ok((hash60, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) + .unwrap(); + + let heights: Vec = cache + .entries + .lock() + .unwrap() + .iter() + .map(|(h, _)| *h) + .collect(); + assert_eq!(heights.len(), 5); + assert!( + !heights.contains(&10), + "lowest cp_height should be evicted, got {:?}", + heights + ); + assert!(heights.contains(&60)); + } + + #[test] + fn insert_skips_values_below_the_cached_minimum() { + let entry_leaves = leaves(2); + let entry_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() + * std::mem::size_of::(); + let cache = full_share_cache(5 * entry_bytes); + let leaf = entry_leaves[0]; + + for cp_height in [10usize, 20, 30, 40, 50] { + let hash = BlockHash::hash(&[cp_height as u8]); + cache + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((hash, leaf)), + || Ok((hash, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) + .unwrap(); + } + assert_eq!(cache.entries.lock().unwrap().len(), 5); + + let hash5 = BlockHash::hash(&[5u8]); + cache + .get_or_build( + 5, + || usize::MAX, + 4, + || Ok((hash5, leaf)), + || Ok((hash5, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) + .unwrap(); + + let heights: Vec = cache + .entries + .lock() + .unwrap() + .iter() + .map(|(h, _)| *h) + .collect(); + assert_eq!(heights.len(), 5); + assert!( + heights.contains(&10), + "a lower cp_height must not displace an existing higher entry, got {:?}", + heights + ); + assert!(!heights.contains(&5)); + } + + #[test] + fn failed_insert_does_not_partially_evict_lower_entries() { + let hash_bytes = std::mem::size_of::(); + let cache = full_share_cache(5 * hash_bytes); + let make_cached = |height: usize, hashes: usize| { + Arc::new(CachedLevels::new( + BlockHash::hash(&[height as u8]), + vec![vec![Sha256dHash::hash(&[height as u8]); hashes]], + )) + }; + + cache.insert(10, usize::MAX, make_cached(10, 1)); + cache.insert(30, usize::MAX, make_cached(30, 4)); + cache.insert(20, usize::MAX, make_cached(20, 2)); + + let heights: Vec = cache + .entries + .lock() + .unwrap() + .iter() + .map(|(height, _)| *height) + .collect(); + assert_eq!(heights, vec![10, 30]); + } - // Single test so the shared static isn't touched by other tests running - // concurrently in the same process. #[test] - fn checkpoint_proof_guard_caps_concurrency_and_releases_on_drop() { + fn insert_caps_entry_count_even_under_a_generous_byte_budget() { + let entry_leaves = leaves(2); + let entry_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() + * std::mem::size_of::(); + let cache = full_share_cache((MAX_CACHE_ENTRIES + 100) * entry_bytes); + let leaf = entry_leaves[0]; + let hash = BlockHash::hash(&[7u8]); + + for cp_height in 0..MAX_CACHE_ENTRIES + 50 { + cache + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((hash, leaf)), + || Ok((hash, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) + .unwrap(); + } + + let len = cache.entries.lock().unwrap().len(); + assert!( + len <= MAX_CACHE_ENTRIES, + "entry count exceeded the cap: {}", + len + ); + } + + #[test] + fn insert_prunes_entries_orphaned_by_a_shorter_reorg() { + let entry_leaves = leaves(2); + let entry_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() + * std::mem::size_of::(); + let leaf = entry_leaves[0]; + let cache = full_share_cache(entry_bytes); + + let old_height = 100; + let old_hash = BlockHash::hash(&[1u8]); + cache + .get_or_build( + old_height, + || old_height, + 4, + || Ok((old_hash, leaf)), + || Ok((old_hash, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) + .unwrap(); + assert_eq!(cache.entries.lock().unwrap().len(), 1); + + let new_height = old_height - 1; + let new_hash = BlockHash::hash(&[2u8]); + let build_count = AtomicUsize::new(0); + + cache + .get_or_build( + new_height, + || new_height, + 4, + || Ok((new_hash, leaf)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + Ok((new_hash, build_levels_above_leaves(&entry_leaves)?, leaf)) + }, + ) + .unwrap(); + assert_eq!(build_count.load(Ordering::SeqCst), 1); + + cache + .get_or_build( + new_height, + || new_height, + 4, + || Ok((new_hash, leaf)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + panic!("must serve from cache instead of rebuilding") + }, + ) + .unwrap(); + assert_eq!( + build_count.load(Ordering::SeqCst), + 1, + "the post-reorg checkpoint must be cached once built, not rebuilt on every request \ + just because the old, now-unreachable height still occupied the budget" + ); + + let heights: Vec = cache + .entries + .lock() + .unwrap() + .iter() + .map(|(h, _)| *h) + .collect(); + assert_eq!(heights, vec![new_height]); + } + + #[test] + fn insertion_reads_best_height_after_a_slow_rebuild() { + let entry_leaves = leaves(2); + let levels = build_levels_above_leaves(&entry_leaves).unwrap(); + let entry_bytes = levels[0].len() * std::mem::size_of::(); + let cache = full_share_cache(2 * entry_bytes); + let leaf = entry_leaves[0]; + + let new_tip_height = 101; + cache.insert( + new_tip_height, + new_tip_height, + Arc::new(CachedLevels::new( + BlockHash::hash(&[new_tip_height as u8]), + levels.clone(), + )), + ); + + let best_height = AtomicUsize::new(new_tip_height - 1); + let cp_height = new_tip_height - 1; + let cp_hash = BlockHash::hash(&[cp_height as u8]); + cache + .get_or_build( + cp_height, + || best_height.load(Ordering::SeqCst), + 1, + || Ok((cp_hash, leaf)), + || { + best_height.store(new_tip_height, Ordering::SeqCst); + Ok((cp_hash, levels, leaf)) + }, + ) + .unwrap(); + + let heights: Vec = cache + .entries + .lock() + .unwrap() + .iter() + .map(|(height, _)| *height) + .collect(); + assert!( + heights.contains(&new_tip_height), + "a tip cached during the rebuild must not be pruned using the rebuild's starting height" + ); + assert!(heights.contains(&cp_height)); + } + + #[test] + fn concurrent_rebuilds_of_same_cp_height_run_build_once() { + let leaves = Arc::new(leaves(9)); + let cp_height = 8; + let cp_hash = BlockHash::hash(&[9u8]); + let sibling = sibling_leaf(&leaves, cp_height, 0); + + let cache = Arc::new(full_share_cache(GENEROUS_TEST_CACHE_BYTES)); + let build_count = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(Barrier::new(8)); + + let handles: Vec<_> = (0..8) + .map(|_| { + let cache = cache.clone(); + let build_count = build_count.clone(); + let barrier = barrier.clone(); + let leaves = leaves.clone(); + thread::spawn(move || { + barrier.wait(); + cache + .get_or_build( + cp_height, + || usize::MAX, + 8, + || Ok((cp_hash, sibling)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + thread::sleep(std::time::Duration::from_millis(20)); + Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)) + }, + ) + .unwrap(); + }) + }) + .collect(); + + for h in handles { + h.join().unwrap(); + } + + assert_eq!( + build_count.load(Ordering::SeqCst), + 1, + "8 concurrent requests for the same cp_height must build exactly once" + ); + assert!( + cache.build_locks.lock().unwrap().is_empty(), + "no build slot may remain after all requests finished" + ); + } + + #[test] + fn concurrent_requests_for_an_uncacheable_height_still_build_once() { + let leaves = Arc::new(leaves(9)); + let cp_height = 8; + let cp_hash = BlockHash::hash(&[9u8]); + let sibling = sibling_leaf(&leaves, cp_height, 0); + + let cache = Arc::new(full_share_cache(0)); + let build_count = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(Barrier::new(8)); + let _keep_build_slot_alive = cache.build_lock_for(cp_height); + + let handles: Vec<_> = (0..8) + .map(|_| { + let cache = cache.clone(); + let build_count = build_count.clone(); + let barrier = barrier.clone(); + let leaves = leaves.clone(); + thread::spawn(move || { + barrier.wait(); + cache + .get_or_build( + cp_height, + || usize::MAX, + 8, + || Ok((cp_hash, sibling)), + || { + build_count.fetch_add(1, Ordering::SeqCst); + thread::sleep(std::time::Duration::from_millis(20)); + Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)) + }, + ) + .unwrap(); + }) + }) + .collect(); + + for h in handles { + h.join().unwrap(); + } + + assert_eq!( + build_count.load(Ordering::SeqCst), + 1, + "8 concurrent requests for the same uncacheable cp_height must still build exactly once" + ); + drop(_keep_build_slot_alive); + assert!( + cache.build_locks.lock().unwrap().is_empty(), + "no build slot may remain once the last request releases it" + ); + } + + #[test] + fn concurrent_releases_must_not_leak_the_build_slot_and_its_tree() { + let leaves = leaves(9); + let cp_height = 8; + let cp_hash = BlockHash::hash(&[9u8]); + let sibling = sibling_leaf(&leaves, cp_height, 0); + + let cache = full_share_cache(0); + + let slot_a = cache.build_lock_for(cp_height); + let slot_b = cache.build_lock_for(cp_height); + + cache + .build_under_lock( + cp_height, + || usize::MAX, + cp_hash, + sibling, + 1, + &slot_a.slot, + || Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)), + ) + .unwrap(); + + drop(slot_a); + assert_eq!( + cache.build_locks.lock().unwrap().len(), + 1, + "slot must stay while another request still holds it" + ); + drop(slot_b); + + let locks = cache.build_locks.lock().unwrap(); + assert!( + locks.is_empty(), + "build slot leaked after all requests finished, retaining the tree \ + outside the cache's memory accounting" + ); + } + + #[test] + fn tree_retained_only_by_a_build_slot_counts_against_the_memory_budget() { + let entry_leaves = leaves(2); + let retained_bytes = build_levels_above_leaves(&entry_leaves).unwrap()[0].len() + * std::mem::size_of::(); + let leaf = entry_leaves[0]; + + let blocked_height = 1_000usize; + let cache = full_share_cache(estimated_peak_build_bytes(blocked_height)); + + let uncached_height = 100; + let uncached_hash = BlockHash::hash(&[1u8]); + let slot_guard = cache.build_lock_for(uncached_height); + cache + .build_under_lock( + uncached_height, + || uncached_height - 1, + uncached_hash, + leaf, + 4, + &slot_guard.slot, + || { + Ok(( + uncached_hash, + build_levels_above_leaves(&entry_leaves)?, + leaf, + )) + }, + ) + .unwrap(); + assert!(cache.entries.lock().unwrap().is_empty()); + assert_eq!( + cache.inflight_build_bytes.load(Ordering::SeqCst), + retained_bytes, + "a tree retained only by last_built must be charged to the budget" + ); + + let blocked_hash = BlockHash::hash(&[2u8]); + let result = cache.get_or_build( + blocked_height, + || usize::MAX, + 4, + || Ok((blocked_hash, leaf)), + || panic!("must not build while retained slot bytes exhaust the budget"), + ); + assert!( + result.is_err(), + "a rebuild must be rejected while slot-retained bytes exhaust the budget" + ); + + drop(slot_guard); + assert_eq!( + cache.inflight_build_bytes.load(Ordering::SeqCst), + 0, + "releasing the slot must release its memory charge" + ); + cache + .get_or_build( + blocked_height, + || usize::MAX, + 4, + || Ok((blocked_hash, leaf)), + || Ok((blocked_hash, build_levels_above_leaves(&entry_leaves)?, leaf)), + ) + .unwrap(); + } + + #[test] + fn a_panicking_build_does_not_poison_the_slot_for_waiters() { + let leaves = Arc::new(leaves(5)); + let cp_height = 4; + let cp_hash = BlockHash::hash(&[1u8]); + let sibling = sibling_leaf(&leaves, cp_height, 0); + + let cache = Arc::new(full_share_cache(GENEROUS_TEST_CACHE_BYTES)); + let _keep_slot_alive = cache.build_lock_for(cp_height); + + let panicking = { + let cache = cache.clone(); + thread::spawn(move || { + let _ = cache.get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((cp_hash, sibling)), + || panic!("simulated build failure"), + ); + }) + }; + assert!(panicking.join().is_err()); + assert_eq!( + cache.build_locks.lock().unwrap().len(), + 1, + "the slot must survive for the still-registered request" + ); + + cache + .get_or_build( + cp_height, + || usize::MAX, + 4, + || Ok((cp_hash, sibling)), + || Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)), + ) + .expect("a waiter must recover from a poisoned slot instead of panicking"); + } + + #[test] + fn rebuild_permit_caps_concurrency_and_releases_on_drop() { const LIMIT: usize = 2; - let mut guards = Vec::new(); + let counter = AtomicUsize::new(0); + let mut permits = Vec::new(); for _ in 0..LIMIT { - guards.push(InflightCheckpointProofGuard::acquire(LIMIT).unwrap()); + permits.push(RebuildPermit::acquire(&counter, LIMIT).unwrap()); } - // All permits are taken: the next caller must be rejected outright - // instead of blocking or panicking. - assert!(InflightCheckpointProofGuard::acquire(LIMIT).is_err()); + assert!(RebuildPermit::acquire(&counter, LIMIT).is_err()); - // Releasing one permit (drop, e.g. on early return via `?`) must let - // the next acquire through. - guards.pop(); - let extra = InflightCheckpointProofGuard::acquire(LIMIT).unwrap(); - - // Back at the cap: still no free permits. - assert!(InflightCheckpointProofGuard::acquire(LIMIT).is_err()); + permits.pop(); + let extra = RebuildPermit::acquire(&counter, LIMIT).unwrap(); + assert!(RebuildPermit::acquire(&counter, LIMIT).is_err()); drop(extra); - drop(guards); + drop(permits); + } + + #[test] + fn cache_hits_bypass_the_rebuild_permit_entirely() { + let leaves = leaves(3); + let cp_height = 2; + let sibling = sibling_leaf(&leaves, cp_height, 0); + let cp_hash = BlockHash::hash(&[3u8]); + + let cache = full_share_cache(GENEROUS_TEST_CACHE_BYTES); + cache + .get_or_build( + cp_height, + || usize::MAX, + 1, + || Ok((cp_hash, sibling)), + || Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)), + ) + .unwrap(); + + cache + .get_or_build( + cp_height, + || usize::MAX, + 0, + || Ok((cp_hash, sibling)), + || panic!("must not rebuild on a cache hit"), + ) + .unwrap(); + } + + #[test] + fn cache_hit_succeeds_while_all_rebuild_permits_are_held() { + let leaves = leaves(3); + let cp_height = 2; + let sibling = sibling_leaf(&leaves, cp_height, 0); + let cp_hash = BlockHash::hash(&[3u8]); + + let cache = Arc::new(full_share_cache(GENEROUS_TEST_CACHE_BYTES)); + cache + .get_or_build( + cp_height, + || usize::MAX, + 1, + || Ok((cp_hash, sibling)), + || Ok((cp_hash, build_levels_above_leaves(&leaves)?, sibling)), + ) + .unwrap(); + + const HOLDERS: usize = 2; + let inside_build = Arc::new(Barrier::new(HOLDERS + 1)); + let release_build = Arc::new(Barrier::new(HOLDERS + 1)); + + let handles: Vec<_> = (0..HOLDERS) + .map(|i| { + let cache = cache.clone(); + let inside_build = inside_build.clone(); + let release_build = release_build.clone(); + thread::spawn(move || { + let miss_height = 1_000 + i; + let hash = BlockHash::hash(&[100 + i as u8]); + let leaf = Sha256dHash::hash(&[0u8]); + cache + .get_or_build( + miss_height, + || usize::MAX, + HOLDERS, + || Ok((hash, leaf)), + || { + inside_build.wait(); + release_build.wait(); + Ok((hash, build_levels_above_leaves(&[leaf])?, leaf)) + }, + ) + .unwrap(); + }) + }) + .collect(); + + inside_build.wait(); + + cache + .get_or_build( + cp_height, + || usize::MAX, + 0, + || Ok((cp_hash, sibling)), + || panic!("must not rebuild on a cache hit"), + ) + .unwrap(); + + release_build.wait(); + for h in handles { + h.join().unwrap(); + } + } + + #[test] + fn memory_permit_caps_concurrent_rebuild_bytes_independently_of_concurrency_limit() { + const HOLDERS: usize = 2; + let holder_heights = [1_000usize, 2_000usize]; + let capacity_bytes: usize = holder_heights + .iter() + .map(|h| estimated_peak_build_bytes(*h)) + .sum(); + + let cache = Arc::new(full_share_cache(capacity_bytes)); + let inside_build = Arc::new(Barrier::new(HOLDERS + 1)); + let release_build = Arc::new(Barrier::new(HOLDERS + 1)); + + let handles: Vec<_> = holder_heights + .iter() + .map(|&miss_height| { + let cache = cache.clone(); + let inside_build = inside_build.clone(); + let release_build = release_build.clone(); + thread::spawn(move || { + let hash = BlockHash::hash(&[miss_height as u8]); + let leaf = Sha256dHash::hash(&[0u8]); + cache + .get_or_build( + miss_height, + || usize::MAX, + 100, + || Ok((hash, leaf)), + || { + inside_build.wait(); + release_build.wait(); + Ok((hash, build_levels_above_leaves(&[leaf])?, leaf)) + }, + ) + .unwrap(); + }) + }) + .collect(); + + inside_build.wait(); + + let result = cache.get_or_build( + 3_000, + || usize::MAX, + 100, + || Ok((BlockHash::hash(&[9u8]), Sha256dHash::hash(&[0u8]))), + || panic!("must not rebuild once the memory budget is fully held"), + ); + assert!( + result.is_err(), + "a third concurrent rebuild must be rejected once the memory budget is exhausted, even with a generous concurrency_limit" + ); + + release_build.wait(); + for h in handles { + h.join().unwrap(); + } + } + + #[test] + fn memory_permit_budget_accounts_for_already_resident_cache_bytes() { + let resident_leaves = leaves(2); + let resident_bytes = build_levels_above_leaves(&resident_leaves).unwrap()[0].len() + * std::mem::size_of::(); + let resident_height = 500usize; + let resident_hash = BlockHash::hash(&[7u8]); + let resident_sibling = resident_leaves[0]; + + let holder_height = 1_000usize; + let extra_height = 2_000usize; + let capacity_bytes = resident_bytes + + estimated_peak_build_bytes(holder_height) + + estimated_peak_build_bytes(extra_height) + - 1; + + let cache = Arc::new(full_share_cache(capacity_bytes)); + cache + .get_or_build( + resident_height, + || usize::MAX, + 1, + || Ok((resident_hash, resident_sibling)), + || { + Ok(( + resident_hash, + build_levels_above_leaves(&resident_leaves)?, + resident_sibling, + )) + }, + ) + .unwrap(); + + let inside_build = Arc::new(Barrier::new(2)); + let release_build = Arc::new(Barrier::new(2)); + + let holder = { + let cache = cache.clone(); + let inside_build = inside_build.clone(); + let release_build = release_build.clone(); + thread::spawn(move || { + let hash = BlockHash::hash(&[9u8]); + let leaf = Sha256dHash::hash(&[0u8]); + cache + .get_or_build( + holder_height, + || usize::MAX, + 100, + || Ok((hash, leaf)), + || { + inside_build.wait(); + release_build.wait(); + Ok((hash, build_levels_above_leaves(&[leaf])?, leaf)) + }, + ) + .unwrap(); + }) + }; + + inside_build.wait(); + + let result = cache.get_or_build( + extra_height, + || usize::MAX, + 100, + || Ok((BlockHash::hash(&[11u8]), Sha256dHash::hash(&[0u8]))), + || panic!("must not rebuild once resident cache bytes and the held build already exhaust the budget"), + ); + assert!( + result.is_err(), + "already-resident cache bytes must count against the budget available to concurrent rebuilds" + ); + + release_build.wait(); + holder.join().unwrap(); + } + + #[test] + fn a_full_cache_still_admits_concurrent_rebuilds() { + let build_height = 1_000usize; + let capacity_bytes = 4 * estimated_peak_build_bytes(build_height); + let cache = Arc::new(CheckpointMerkleCache::new(capacity_bytes)); + + let filler_leaves = leaves(1 << 12); + let filler_bytes = CachedLevels::new( + BlockHash::hash(&[0u8]), + build_levels_above_leaves(&filler_leaves).unwrap(), + ) + .size_bytes; + let mut height = 1usize << 20; + let mut resident = 0usize; + while resident + filler_bytes <= capacity_bytes / 2 { + let hash = BlockHash::hash(&height.to_le_bytes()); + cache + .get_or_build( + height, + || usize::MAX, + 1, + || Ok((hash, filler_leaves[0])), + || Ok((hash, build_levels_above_leaves(&filler_leaves)?, filler_leaves[0])), + ) + .unwrap(); + resident += filler_bytes; + height += 1; + } + assert!(resident > 0); + + let entered = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let handles: Vec<_> = vec![build_height, build_height + 1] + .into_iter() + .map(|miss_height: usize| { + let cache = cache.clone(); + let entered = entered.clone(); + let release = release.clone(); + thread::spawn(move || { + let hash = BlockHash::hash(&[miss_height as u8]); + let leaf = Sha256dHash::hash(&[0u8]); + cache.get_or_build( + miss_height, + || usize::MAX, + 100, + || Ok((hash, leaf)), + || { + entered.fetch_add(1, Ordering::SeqCst); + while !release.load(Ordering::SeqCst) { + thread::sleep(std::time::Duration::from_millis(1)); + } + Ok((hash, build_levels_above_leaves(&[leaf])?, leaf)) + }, + ) + }) + }) + .collect(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while entered.load(Ordering::SeqCst) < 2 + && !handles.iter().any(|handle| handle.is_finished()) + && std::time::Instant::now() < deadline + { + thread::sleep(std::time::Duration::from_millis(1)); + } + release.store(true, Ordering::SeqCst); + for handle in handles { + assert!( + handle.join().unwrap().is_ok(), + "cached trees must not leave room for only a single rebuild" + ); + } } } diff --git a/tests/common.rs b/tests/common.rs index 7bcd19d82..3ca849501 100644 --- a/tests/common.rs +++ b/tests/common.rs @@ -24,7 +24,7 @@ use electrs::{ daemon::Daemon, electrum::RPC as ElectrumRPC, metrics::Metrics, - new_index::{ChainQuery, FetchFrom, Indexer, Mempool, Query, Store}, + new_index::{ChainQuery, FetchFrom, Indexer, Mempool, MempoolSyncStatus, Query, Store}, rest, signal::Waiter, }; @@ -45,6 +45,10 @@ pub struct TestRunner { impl TestRunner { pub fn new() -> Result { + Self::new_with(|_| {}) + } + + pub fn new_with(configure: impl FnOnce(&mut Config)) -> Result { let log = init_log(); // Setup the bitcoind/elementsd config @@ -89,7 +93,7 @@ impl TestRunner { let electrsdb = tempfile::tempdir().unwrap(); - let config = Arc::new(Config { + let mut config = Config { log, network_type, db_path: electrsdb.path().to_path_buf(), @@ -102,7 +106,16 @@ impl TestRunner { cookie: None, electrum_rpc_addr: rand_available_addr(), electrum_rpc_conn_max_age: None, + electrum_rpc_write_timeout: Some(std::time::Duration::from_secs(30)), electrum_rpc_max_request_num_bytes: 1_048_576, + #[cfg(not(feature = "liquid"))] + electrum_rpc_max_response_num_bytes: 8_388_608, + #[cfg(feature = "liquid")] + electrum_rpc_max_response_num_bytes: 33_554_432, + #[cfg(not(feature = "liquid"))] + electrum_rpc_global_response_budget_bytes: 67_108_864, + #[cfg(feature = "liquid")] + electrum_rpc_global_response_budget_bytes: 268_435_456, http_addr: rand_available_addr(), http_socket_file: None, // XXX test with socket file or tcp? monitoring_addr: rand_available_addr(), @@ -114,9 +127,12 @@ impl TestRunner { cors: None, precache_scripts: None, utxos_limit: 100, + history_scan_limit: 100_000, + utxos_checkpoint_limit: 1_000, electrum_txs_limit: 100, electrum_subscription_limit: 10_000, electrum_checkpoint_proof_concurrency_limit: 2, + electrum_checkpoint_merkle_cache_mb: 256, electrum_banner: "".into(), rpc_logging: RpcLogging::default(), zmq_addr: None, @@ -136,7 +152,9 @@ impl TestRunner { //electrum_announce: bool, //#[cfg(feature = "electrum-discovery")] //tor_proxy: Option, - }); + }; + configure(&mut config); + let config = Arc::new(config); let signal = Waiter::start(crossbeam_channel::never()); let metrics_addr = rand_available_addr(); @@ -185,7 +203,10 @@ impl TestRunner { &metrics, Arc::clone(&config), ))); - assert!(Mempool::update(&mempool, &daemon, &tip)?); + assert_eq!( + Mempool::update(&mempool, &daemon, &tip)?, + MempoolSyncStatus::Synced + ); let query = Arc::new(Query::new( Arc::clone(&chain), @@ -212,10 +233,6 @@ impl TestRunner { }) } - pub fn query(&self) -> Arc { - Arc::clone(&self.query) - } - pub fn metrics_addr(&self) -> net::SocketAddr { self.metrics_addr } @@ -227,9 +244,28 @@ impl TestRunner { return &self.node.client(); } + pub fn config(&self) -> &Arc { + &self.config + } + + pub fn query(&self) -> &Arc { + &self.query + } + + pub fn mempool(&self) -> &Arc> { + &self.mempool + } + + pub fn daemon(&self) -> &Arc { + &self.daemon + } + pub fn sync(&mut self) -> Result<()> { let tip = self.indexer.update(&self.daemon)?; - assert!(Mempool::update(&self.mempool, &self.daemon, &tip)?); + assert_eq!( + Mempool::update(&self.mempool, &self.daemon, &tip)?, + MempoolSyncStatus::Synced + ); // force an update for the mempool stats, which are normally cached self.mempool.write().unwrap().update_backlog_stats(); Ok(()) @@ -251,6 +287,94 @@ impl TestRunner { Ok(txid) } + #[cfg(not(feature = "liquid"))] + pub fn send_multi( + &mut self, + addr: &Address, + amount: bitcoin::Amount, + count: usize, + ) -> Result { + let fee = bitcoin::Amount::from_sat(50_000); + let total_out = bitcoin::Amount::from_sat(amount.to_sat() * count as u64); + + let utxos: Vec = self.node_client().call("listunspent", &[])?; + let utxo = utxos + .iter() + .max_by(|a, b| { + a["amount"] + .as_f64() + .unwrap() + .total_cmp(&b["amount"].as_f64().unwrap()) + }) + .expect("wallet has no spendable utxos"); + let input_value = bitcoin::Amount::from_btc(utxo["amount"].as_f64().unwrap()).unwrap(); + let change = input_value - total_out - fee; + + let input = bitcoin::TxIn { + previous_output: bitcoin::OutPoint { + txid: utxo["txid"].as_str().unwrap().parse().unwrap(), + vout: utxo["vout"].as_u64().unwrap() as u32, + }, + ..Default::default() + }; + let mut outputs: Vec = (0..count) + .map(|_| bitcoin::TxOut { + value: amount, + script_pubkey: addr.script_pubkey(), + }) + .collect(); + outputs.push(bitcoin::TxOut { + value: change, + script_pubkey: self.newaddress()?.script_pubkey(), + }); + + let tx = bitcoin::Transaction { + version: bitcoin::transaction::Version::TWO, + lock_time: bitcoin::absolute::LockTime::ZERO, + input: vec![input], + output: outputs, + }; + let raw_hex = bitcoin::consensus::encode::serialize_hex(&tx); + let signed: serde_json::Value = self + .node_client() + .call("signrawtransactionwithwallet", &[json!(raw_hex)])?; + let signed_hex = signed["hex"].as_str().unwrap().to_string(); + let txid: Txid = self + .node_client() + .call("sendrawtransaction", &[json!(signed_hex)])?; + self.sync()?; + Ok(txid) + } + + #[cfg(not(feature = "liquid"))] + pub fn consolidate( + &mut self, + txid: Txid, + n_outputs: usize, + amount_per_output: bitcoin::Amount, + fee: bitcoin::Amount, + to: &Address, + ) -> Result { + let inputs: Vec = (0..n_outputs as u32) + .map(|vout| json!({ "txid": txid.to_string(), "vout": vout })) + .collect(); + let total = bitcoin::Amount::from_sat(amount_per_output.to_sat() * n_outputs as u64); + let output_amount = bitcoin::Amount::from_sat(total.to_sat() - fee.to_sat()); + let outputs = json!({ to.to_string(): output_amount.to_btc() }); + let raw_hex: String = self + .node_client() + .call("createrawtransaction", &[json!(inputs), outputs])?; + let signed: serde_json::Value = self + .node_client() + .call("signrawtransactionwithwallet", &[json!(raw_hex)])?; + let signed_hex = signed["hex"].as_str().unwrap().to_string(); + let txid: Txid = self + .node_client() + .call("sendrawtransaction", &[json!(signed_hex)])?; + self.sync()?; + Ok(txid) + } + #[cfg(feature = "liquid")] pub fn send_asset( &mut self, @@ -334,17 +458,55 @@ impl TestRunner { .transaction() .unwrap()) } + + /// Number of client-proxied daemon RPCs that completed with the given `result` label. + pub fn proxied_rpc_count(&self, result: &str) -> f64 { + self.metrics + .gather() + .iter() + .filter(|family| family.name() == "daemon_rpc_proxied") + .flat_map(|family| family.get_metric()) + .filter(|metric| { + metric + .get_label() + .iter() + .any(|label| label.name() == "result" && label.value() == result) + }) + .map(|metric| metric.get_counter().value()) + .sum() + } + + pub fn counter_value(&self, name: &str) -> f64 { + self.metrics + .gather() + .iter() + .filter(|family| family.name() == name) + .flat_map(|family| family.get_metric()) + .map(|metric| metric.get_counter().value()) + .sum() + } } pub fn init_rest_tester() -> Result<(rest::Handle, net::SocketAddr, TestRunner)> { - let tester = TestRunner::new()?; + init_rest_tester_with(|_| {}) +} +pub fn init_rest_tester_with( + configure: impl FnOnce(&mut Config), +) -> Result<(rest::Handle, net::SocketAddr, TestRunner)> { + let tester = TestRunner::new_with(configure)?; let addr = tester.config.http_addr; let rest_server = rest::start(Arc::clone(&tester.config), Arc::clone(&tester.query)); wait_for_tcp(addr, "REST"); Ok((rest_server, addr, tester)) } pub fn init_electrum_tester() -> Result<(ElectrumRPC, net::SocketAddr, TestRunner)> { - let tester = TestRunner::new()?; + init_electrum_tester_with_config(|_| {}) +} + +pub fn init_electrum_tester_with_config( + configure: impl FnOnce(&mut Config), +) -> Result<(ElectrumRPC, net::SocketAddr, TestRunner)> { + let tester = TestRunner::new_with(configure)?; let addr = tester.config.electrum_rpc_addr; let electrum_server = ElectrumRPC::start( Arc::clone(&tester.config), diff --git a/tests/config.rs b/tests/config.rs index 82e8f1eec..2962177c2 100644 --- a/tests/config.rs +++ b/tests/config.rs @@ -80,3 +80,103 @@ fn startup_debug_log_identifies_cookie_file() { stderr ); } + +#[test] +fn checkpoint_proof_concurrency_limit_defaults_to_half_the_cpu_cores() { + let temp_dir = tempfile::tempdir().unwrap(); + let output = run_electrs(temp_dir.path(), &["-vv"]); + let stderr = String::from_utf8(output.stderr).unwrap(); + + let expected = (num_cpus::get() / 2).max(1); + let needle = format!("electrum_checkpoint_proof_concurrency_limit: {}", expected); + assert!( + stderr.contains(&needle), + "expected '{}' in stderr, got: {}", + needle, + stderr + ); +} + +#[test] +fn checkpoint_proof_concurrency_limit_can_be_overridden() { + let temp_dir = tempfile::tempdir().unwrap(); + let output = run_electrs( + temp_dir.path(), + &["-vv", "--electrum-checkpoint-proof-concurrency-limit", "7"], + ); + let stderr = String::from_utf8(output.stderr).unwrap(); + + assert!( + stderr.contains("electrum_checkpoint_proof_concurrency_limit: 7"), + "explicit override was not honored, stderr: {}", + stderr + ); +} + +fn assert_rejected(extra_args: &[&str], needle: &str) { + let temp_dir = tempfile::tempdir().unwrap(); + let output = run_electrs(temp_dir.path(), extra_args); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!( + !output.status.success(), + "electrs accepted {:?}", + extra_args + ); + assert!( + stderr.contains(needle), + "expected '{}' in stderr, got: {}", + needle, + stderr + ); +} + +#[test] +fn zero_history_scan_limit_is_rejected() { + assert_rejected( + &["--history-scan-limit", "0"], + "--history-scan-limit must be at least 1", + ); +} + +#[test] +fn utxos_checkpoint_limit_below_utxos_limit_is_rejected() { + assert_rejected( + &["--utxos-limit", "500", "--utxos-checkpoint-limit", "499"], + "--utxos-checkpoint-limit (499) must be >= --utxos-limit (500)", + ); +} + +#[test] +fn disabled_write_timeout_requires_an_unlimited_budget_or_max_age() { + assert_rejected( + &["--electrum-rpc-write-timeout", "0"], + "requires a non-zero --electrum-rpc-write-timeout or --electrum-rpc-conn-max-age", + ); + + for extra_args in [ + [ + "--electrum-rpc-write-timeout", + "0", + "--electrum-rpc-global-response-budget-bytes", + "0", + ], + [ + "--electrum-rpc-write-timeout", + "0", + "--electrum-rpc-conn-max-age", + "600", + ], + ] { + let temp_dir = tempfile::tempdir().unwrap(); + let mut args = vec!["-vv"]; + args.extend(extra_args); + let output = run_electrs(temp_dir.path(), &args); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!( + stderr.contains("electrum_rpc_write_timeout: None"), + "{:?} should pass validation, stderr: {}", + extra_args, + stderr + ); + } +} diff --git a/tests/electrum.rs b/tests/electrum.rs index 3f064f575..da8db0188 100644 --- a/tests/electrum.rs +++ b/tests/electrum.rs @@ -445,6 +445,255 @@ fn test_electrum_broadcast_package_updates_mempool() -> Result<()> { Ok(()) } +#[cfg(not(feature = "liquid"))] +mod checkpoint { + use bitcoin::hashes::{sha256d, Hash}; + + pub fn merklize(left: sha256d::Hash, right: sha256d::Hash) -> sha256d::Hash { + let mut data = [0u8; 64]; + data[..32].copy_from_slice(&left[..]); + data[32..].copy_from_slice(&right[..]); + sha256d::Hash::hash(&data) + } + + pub fn merkle_root(mut hashes: Vec) -> sha256d::Hash { + assert!(!hashes.is_empty()); + while hashes.len() > 1 { + if hashes.len() % 2 != 0 { + hashes.push(*hashes.last().unwrap()); + } + hashes = hashes + .chunks(2) + .map(|pair| merklize(pair[0], pair[1])) + .collect(); + } + hashes[0] + } + + pub fn root_from_branch( + leaf: sha256d::Hash, + branch: &[sha256d::Hash], + mut index: usize, + ) -> sha256d::Hash { + let mut current = leaf; + for sibling in branch { + current = if index % 2 == 0 { + merklize(current, *sibling) + } else { + merklize(*sibling, current) + }; + index /= 2; + } + current + } +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_electrum_checkpoint_proofs() -> Result<()> { + use bitcoin::hashes::{sha256d, Hash}; + use bitcoin::hex::FromHex; + use std::str::FromStr; + + let (_electrum_server, electrum_addr, tester) = common::init_electrum_tester()?; + let tip = tester.get_block_count()? as usize; + + let leaves: Vec = (0..=tip) + .map(|h| Ok(tester.get_block_hash(h as u64)?.to_raw_hash())) + .collect::>()?; + let expected_root = checkpoint::merkle_root(leaves.clone()); + + let mut stream = TcpStream::connect(electrum_addr).unwrap(); + let parse = |s: &str| -> electrumd::jsonrpc::serde_json::Value { + electrumd::jsonrpc::serde_json::from_str(s).unwrap() + }; + + let height = 5usize; + let v = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.header\", \"params\": [{}, {}], \"id\": 1}}", + height, tip + ), + )); + let result = &v["result"]; + let header_bytes = + Vec::::from_hex(result["header"].as_str().expect("header hex")).unwrap(); + assert_eq!( + sha256d::Hash::hash(&header_bytes), + leaves[height], + "returned header must hash to the block hash at the requested height" + ); + let root = sha256d::Hash::from_str(result["root"].as_str().expect("root")).unwrap(); + assert_eq!(root, expected_root, "proof root must match the independent tree"); + let branch: Vec = result["branch"] + .as_array() + .expect("branch array") + .iter() + .map(|h| sha256d::Hash::from_str(h.as_str().unwrap()).unwrap()) + .collect(); + assert_eq!( + checkpoint::root_from_branch(leaves[height], &branch, height), + expected_root, + "branch must connect the leaf to the root" + ); + + let v2 = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.header\", \"params\": [{}, {}], \"id\": 2}}", + height, tip + ), + )); + assert_eq!(v["result"], v2["result"], "cached proof must be identical"); + + let v = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.header\", \"params\": [{}, 0], \"id\": 3}}", + height + ), + )); + assert!(v["result"].is_string(), "cp_height=0 must return bare hex"); + + let v = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.header\", \"params\": [{}, {}], \"id\": 4}}", + height, height - 1 + ), + )); + assert!(!v["error"].is_null(), "cp_height < height must be an error"); + + let v = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.header\", \"params\": [{}, {}], \"id\": 5}}", + height, tip + 1 + ), + )); + assert!(!v["error"].is_null(), "cp_height above the tip must be an error"); + + let (start, count) = (0usize, 10usize); + let v = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.headers\", \"params\": [{}, {}, {}], \"id\": 6}}", + start, count, tip + ), + )); + let result = &v["result"]; + assert_eq!(result["count"].as_u64(), Some(count as u64)); + let hex = result["hex"].as_str().expect("headers hex"); + assert_eq!(hex.len(), count * 80 * 2); + let root = sha256d::Hash::from_str(result["root"].as_str().expect("root")).unwrap(); + assert_eq!(root, expected_root); + let branch: Vec = result["branch"] + .as_array() + .expect("branch array") + .iter() + .map(|h| sha256d::Hash::from_str(h.as_str().unwrap()).unwrap()) + .collect(); + let proof_height = start + count - 1; + assert_eq!( + checkpoint::root_from_branch(leaves[proof_height], &branch, proof_height), + expected_root, + "headers branch must connect the last requested header to the root" + ); + + let v = parse(&write_and_read( + &mut stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.headers\", \"params\": [{}, {}, {}], \"id\": 7}}", + tip - 1, 5, tip + ), + )); + assert!( + !v["error"].is_null(), + "a headers range extending past cp_height must be an error" + ); + + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_electrum_checkpoint_proof_reorg_consistency() -> Result<()> { + use bitcoin::hashes::{sha256d, Hash}; + use bitcoin::hex::FromHex; + use std::str::FromStr; + + let (_electrum_server, electrum_addr, mut tester) = common::init_electrum_tester()?; + let tip = tester.get_block_count()? as usize; + let height = 3usize; + + let mut stream = TcpStream::connect(electrum_addr).unwrap(); + let parse = |s: &str| -> electrumd::jsonrpc::serde_json::Value { + electrumd::jsonrpc::serde_json::from_str(s).unwrap() + }; + let request = |stream: &mut TcpStream, id: u32| { + write_and_read( + stream, + &format!( + "{{\"jsonrpc\": \"2.0\", \"method\": \"blockchain.block.header\", \"params\": [{}, {}], \"id\": {}}}", + height, tip, id + ), + ) + }; + + let v = parse(&request(&mut stream, 1)); + let old_root = sha256d::Hash::from_str(v["result"]["root"].as_str().expect("root")).unwrap(); + + let old_tip_hash = tester.get_block_hash(tip as u64)?; + let _: electrumd::jsonrpc::serde_json::Value = tester + .node_client() + .call("invalidateblock", &[json!(old_tip_hash.to_string())])?; + tester.mine()?; + tester.mine()?; + assert_ne!( + tester.get_block_hash(tip as u64)?, + old_tip_hash, + "the reorg must have replaced the block at the old tip height" + ); + + let v = parse(&request(&mut stream, 2)); + let result = &v["result"]; + let new_root = sha256d::Hash::from_str(result["root"].as_str().expect("root")).unwrap(); + assert_ne!( + new_root, old_root, + "a reorg past the checkpoint must invalidate the cached tree" + ); + + let leaves: Vec = (0..=tip) + .map(|h| Ok(tester.get_block_hash(h as u64)?.to_raw_hash())) + .collect::>()?; + assert_eq!( + new_root, + checkpoint::merkle_root(leaves.clone()), + "post-reorg root must match the new chain" + ); + let branch: Vec = result["branch"] + .as_array() + .expect("branch array") + .iter() + .map(|h| sha256d::Hash::from_str(h.as_str().unwrap()).unwrap()) + .collect(); + assert_eq!( + checkpoint::root_from_branch(leaves[height], &branch, height), + new_root, + "post-reorg branch must connect the leaf to the new root" + ); + let header_bytes = + Vec::::from_hex(result["header"].as_str().expect("header hex")).unwrap(); + assert_eq!( + sha256d::Hash::hash(&header_bytes), + leaves[height], + "post-reorg header must belong to the same snapshot as the proof" + ); + + Ok(()) +} + fn write_and_read(stream: &mut TcpStream, write: &str) -> String { stream.write_all(write.as_bytes()).unwrap(); stream.write(b"\n").unwrap(); diff --git a/tests/electrum_response_limits.rs b/tests/electrum_response_limits.rs new file mode 100644 index 000000000..2585e3b93 --- /dev/null +++ b/tests/electrum_response_limits.rs @@ -0,0 +1,291 @@ +#[allow(dead_code)] +mod common; + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{Shutdown, SocketAddr, TcpStream}; +use std::time::{Duration, Instant}; + +use serde_json::{json, Value}; + +struct Client(BufReader); + +impl Client { + fn connect(addr: SocketAddr) -> Self { + let stream = TcpStream::connect(addr).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(15))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(15))) + .unwrap(); + Self(BufReader::new(stream)) + } + + fn send(&mut self, request: &Value) { + serde_json::to_writer(self.0.get_mut(), request).unwrap(); + self.0.get_mut().write_all(b"\n").unwrap(); + } + + fn receive(&mut self) -> (Value, usize) { + let mut line = String::new(); + assert!( + self.0.read_line(&mut line).unwrap() > 0, + "connection closed" + ); + assert!(line.ends_with('\n')); + (serde_json::from_str(&line).unwrap(), line.len()) + } + + fn call(&mut self, request: Value) -> (Value, usize) { + self.send(&request); + self.receive() + } +} + +impl Drop for Client { + fn drop(&mut self) { + let _ = self.0.get_ref().shutdown(Shutdown::Both); + } +} + +#[test] +fn unaligned_budget_admits_replies_through_the_exact_line_limit() { + let cap = 100_000; + let banner = "a".repeat(90_000); + let (_server, addr, _tester) = common::init_electrum_tester_with_config(|config| { + config.electrum_rpc_max_response_num_bytes = cap; + config.electrum_rpc_global_response_budget_bytes = cap; + config.electrum_banner = banner.clone(); + }) + .unwrap(); + let mut client = Client::connect(addr); + let (reply, _) = client.call(json!({"id":1,"method":"server.banner"})); + assert_eq!(reply["result"], banner); + + let base = serde_json::to_vec(&json!({"id":"","jsonrpc":"2.0","result":banner})) + .unwrap() + .len() + + 1; + let id = "b".repeat(cap - base); + let (reply, size) = client.call(json!({"id":id,"method":"server.banner"})); + assert_eq!(size, cap); + assert_eq!(reply["id"], id); + assert_eq!(reply["result"], banner); + + let id = format!("{}c", id); + let (reply, size) = client.call(json!({"id":id,"method":"server.banner"})); + assert!(size <= cap); + assert_eq!(reply["id"], id); + assert_eq!(reply["error"]["code"], 1); + + client.0.get_mut().write_all(b"{\n").unwrap(); + let (reply, size) = client.receive(); + assert!(size <= cap); + assert_eq!(reply["id"], Value::Null); + assert_eq!(reply["error"]["code"], -32700); + let (reply, _) = client.call(json!({"id":2,"method":false})); + assert_eq!(reply["id"], 2); + assert_eq!(reply["error"]["code"], -32600); + let (reply, _) = client.call(json!({"id":3,"method":"server.ping"})); + assert_eq!(reply["result"], Value::Null); + assert!(reply.get("error").is_none()); +} + +#[test] +fn response_line_limits_preserve_ids_and_skip_later_commands() { + let cap = 100_000; + let (_server, addr, _tester) = common::init_electrum_tester_with_config(|config| { + config.electrum_rpc_max_response_num_bytes = cap; + config.electrum_rpc_global_response_budget_bytes = usize::MAX; + config.electrum_banner = "a".repeat(80_000); + }) + .unwrap(); + let mut client = Client::connect(addr); + let (single, len) = client.call(json!({"id":1,"method":"server.banner"})); + assert_eq!(single["result"].as_str().unwrap().len(), 80_000); + assert!(len <= cap); + + let hash = "00".repeat(32); + let (batch, len) = client.call(json!([ + {"id":1,"method":"server.banner"}, + {"id":2,"method":"server.banner"}, + {"id":3,"method":"blockchain.scripthash.subscribe","params":[hash]} + ])); + assert!(len <= cap); + assert_eq!(batch[0]["result"], single["result"]); + assert_eq!(batch[1]["id"], 2); + assert_eq!(batch[1]["error"]["code"], 1); + assert_eq!(batch[2]["id"], 3); + assert_eq!(batch[2]["error"]["code"], 1); + let (unsubscribed, _) = + client.call(json!({"id":4,"method":"blockchain.scripthash.unsubscribe","params":[hash]})); + assert_eq!(unsubscribed["result"], false); + let (empty, len) = client.call(json!([])); + assert_eq!(empty, json!([])); + assert_eq!(len, 3); + + let id = "b".repeat(30_000); + let (overflow, len) = client.call(json!({"id":id,"method":"server.banner"})); + assert_eq!(overflow["id"], id); + assert_eq!(overflow["error"]["code"], 1); + assert!(len <= cap); +} + +#[test] +fn response_budget_recovers_when_stalled_batch_writers_time_out() { + let cap = 16 * 1024 * 1024; + let banner_bytes = 4 * 1024 * 1024 - 4096; + let (_server, addr, tester) = common::init_electrum_tester_with_config(|config| { + config.electrum_rpc_max_response_num_bytes = cap; + config.electrum_rpc_global_response_budget_bytes = cap; + config.electrum_rpc_write_timeout = Some(Duration::from_secs(5)); + config.electrum_rpc_conn_max_age = None; + config.electrum_banner = "a".repeat(banner_bytes); + }) + .unwrap(); + + let mut holders = Vec::new(); + for _ in 0..2 { + // The 8 MiB reply fills the default TCP buffers. + let mut client = Client::connect(addr); + client.send(&json!([ + {"id":1,"method":"server.banner"}, + {"id":2,"method":"server.banner"} + ])); + let mut first = [0]; + client.0.read_exact(&mut first).unwrap(); + assert_eq!(first, [b'[']); + holders.push(client); + } + + let mut healthy = Client::connect(addr); + let request = json!({"id":"x".repeat(20 * 1024),"method":"server.ping"}); + let (rejected, _) = healthy.call(request.clone()); + assert_eq!(rejected["error"]["code"], 2); + + // Keep both clients open and unread. Recovery must come from the server's + // write deadline, with connection-age expiry explicitly disabled. + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let (reply, _) = healthy.call(request.clone()); + if reply.get("result").is_some() { + assert_eq!(reply["result"], Value::Null); + break; + } + assert_eq!(reply["error"]["code"], 2); + assert!( + Instant::now() < deadline, + "stalled writers did not release the budget" + ); + std::thread::sleep(Duration::from_millis(20)); + } + let (reply, _) = healthy.call(json!({"id":3,"method":"server.banner"})); + assert_eq!(reply["result"].as_str().unwrap().len(), banner_bytes); + assert!( + tester.counter_value("electrum_client_write_timeouts_total") >= 1.0, + "budget recovered without a stalled writer timing out" + ); +} + +#[test] +fn normal_close_delivers_final_response_before_eof() { + let banner = "a".repeat(256 * 1024); + let (_server, addr, _tester) = common::init_electrum_tester_with_config(|config| { + config.electrum_banner = banner.clone(); + config.electrum_rpc_write_timeout = Some(Duration::from_secs(5)); + }) + .unwrap(); + let mut client = Client::connect(addr); + client.send(&json!({"id":1,"method":"server.banner"})); + // EOF on the request side makes the server close after its final reply. + client.0.get_ref().shutdown(Shutdown::Write).unwrap(); + let (reply, _) = client.receive(); + assert_eq!(reply["result"], banner); + assert_eq!(client.0.read(&mut [0]).unwrap(), 0); +} + +#[test] +fn response_budget_streams_large_ids_and_recovers_after_blocked_writers() { + let cap = 16 * 1024 * 1024; + let banner_bytes = 8 * 1024 * 1024 - 8192; + let (_server, addr, _tester) = common::init_electrum_tester_with_config(|config| { + config.electrum_rpc_max_response_num_bytes = cap; + config.electrum_rpc_global_response_budget_bytes = cap; + config.electrum_banner = "a".repeat(banner_bytes); + }) + .unwrap(); + + let mut holders = Vec::new(); + for id in 0..2 { + let mut client = Client::connect(addr); + socket2::SockRef::from(client.0.get_ref()) + .set_recv_buffer_size(4096) + .unwrap(); + client.send(&json!({"id":id,"method":"server.banner"})); + // Receiving the start proves assembly finished and its reservation is + // held by write_all. The tiny receive window cannot absorb 8 MiB. + let mut first = [0]; + client.0.read_exact(&mut first).unwrap(); + assert_eq!(first, [b'{']); + holders.push(client); + } + + let mut client = Client::connect(addr); + let id = "x".repeat(20 * 1024); + let hash = "00".repeat(32); + let (error, len) = + client.call(json!({"id":id,"method":"blockchain.scripthash.subscribe","params":[hash]})); + assert_eq!(error["id"], id); + assert_eq!(error["error"]["code"], 2); + assert!(len <= cap); + let (unsubscribed, _) = + client.call(json!({"id":4,"method":"blockchain.scripthash.unsubscribe","params":[hash]})); + assert_eq!(unsubscribed["result"], false); + + let (batch, len) = client.call(json!([ + {"id":id,"method":"server.ping"}, + {"id":5,"method":"blockchain.scripthash.subscribe","params":[hash]} + ])); + assert_eq!(batch[0]["id"], id); + assert_eq!(batch[0]["error"]["code"], 2); + assert_eq!(batch[1]["id"], 5); + assert_eq!(batch[1]["error"]["code"], 2); + assert!(len <= cap); + let (unsubscribed, _) = + client.call(json!({"id":6,"method":"blockchain.scripthash.unsubscribe","params":[hash]})); + assert_eq!(unsubscribed["result"], false); + + // Free one holder, leaving 8 MiB available. A banner fits, and the + // reserved error for the next element must fit without another grant. + drop(holders.pop()); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let (reply, _) = client.call(json!({"id":id,"method":"server.ping"})); + assert_eq!(reply["id"], id); + if reply.get("result").is_some() { + assert_eq!(reply["result"], Value::Null); + break; + } + assert!( + Instant::now() < deadline, + "response budget was not released" + ); + std::thread::sleep(Duration::from_millis(10)); + } + let second_id = "y".repeat(14_000); + let (batch, len) = client.call(json!([ + {"id":7,"method":"server.banner"}, + {"id":second_id,"method":"server.banner"}, + {"id":8,"method":"blockchain.scripthash.subscribe","params":[hash]} + ])); + assert_eq!(batch[0]["result"].as_str().unwrap().len(), banner_bytes); + assert_eq!(batch[1]["id"], second_id); + assert_eq!(batch[1]["error"]["code"], 2); + assert_eq!(batch[2]["id"], 8); + assert_eq!(batch[2]["error"]["code"], 2); + assert!(len <= cap); + let (unsubscribed, _) = + client.call(json!({"id":9,"method":"blockchain.scripthash.unsubscribe","params":[hash]})); + assert_eq!(unsubscribed["result"], false); + drop(holders); +} diff --git a/tests/rest.rs b/tests/rest.rs index 91100cf9c..67287cff3 100644 --- a/tests/rest.rs +++ b/tests/rest.rs @@ -1367,6 +1367,151 @@ fn test_rest_package_updates_mempool() -> Result<()> { Ok(()) } +/// Build two spends of a fresh `OP_DROP OP_TRUE` P2WSH output that share a txid but differ in +/// witness, returning (txid, first variant hex, second variant hex). +#[cfg(not(feature = "liquid"))] +fn same_txid_witness_variants(tester: &mut common::TestRunner) -> Result<(Txid, String, String)> { + use bitcoin::blockdata::opcodes::all::{OP_DROP, OP_PUSHNUM_1}; + use bitcoin::consensus::encode::serialize_hex; + use bitcoin::{ + absolute, transaction, Address, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxOut, + Witness, + }; + + let witness_script = ScriptBuf::builder() + .push_opcode(OP_DROP) + .push_opcode(OP_PUSHNUM_1) + .into_script(); + let p2wsh = Address::p2wsh(&witness_script, bitcoin::Network::Regtest); + let funding_txid = tester.send(&p2wsh, Amount::from_sat(100_000))?; + let funding_tx = tester.get_raw_transaction(funding_txid)?; + tester.mine()?; + let vout = funding_tx + .output + .iter() + .position(|txout| txout.script_pubkey == p2wsh.script_pubkey()) + .expect("funding output") as u32; + + let spend = |witness_item: &[u8]| Transaction { + version: transaction::Version::TWO, + lock_time: absolute::LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::new(funding_txid, vout), + script_sig: ScriptBuf::new(), + sequence: Sequence::ENABLE_RBF_NO_LOCKTIME, + witness: Witness::from_slice(&[witness_item, witness_script.as_bytes()]), + }], + output: vec![TxOut { + value: Amount::from_sat(90_000), + script_pubkey: p2wsh.script_pubkey(), + }], + }; + let first = spend(&[0x01]); + let second = spend(&[0x02; 32]); + assert_eq!(first.compute_txid(), second.compute_txid()); + assert_ne!(first.compute_wtxid(), second.compute_wtxid()); + + Ok(( + first.compute_txid(), + serialize_hex(&first), + serialize_hex(&second), + )) +} + +/// Regression test: when bitcoind already holds a transaction with the same txid but a different +/// witness, broadcasting the other variant succeeds without replacing it. electrs must index the +/// daemon's copy, not the submitted one. +#[cfg(not(feature = "liquid"))] +#[test] +fn test_rest_broadcast_indexes_the_daemons_witness() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap(); + + for endpoint in ["/tx", "/txs/package"] { + let (txid, in_mempool_hex, submitted_hex) = same_txid_witness_variants(&mut tester)?; + let _: Value = tester + .node_client() + .call("sendrawtransaction", &[in_mempool_hex.clone().into()])?; + + let resp = match endpoint { + "/tx" => ureq::post(&format!("http://{}/tx", rest_addr)).send(&submitted_hex)?, + _ => ureq::post(&format!("http://{}/txs/package", rest_addr)) + .send_json([&submitted_hex])?, + }; + assert_eq!(resp.status(), 200, "{} rejected the variant", endpoint); + + assert_eq!( + get_plain(rest_addr, &format!("/tx/{}/hex", txid))?, + in_mempool_hex, + "{} indexed the submitted witness instead of the daemon's", + endpoint + ); + } + + rest_handle.stop(); + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_mempool_ensure_txs_concurrency() -> Result<()> { + use electrs::new_index::Mempool; + use std::sync::{mpsc, Arc}; + use std::thread; + use std::time::{Duration, Instant}; + + const TIMEOUT: Duration = Duration::from_secs(10); + + let (rest_handle, rest_addr, tester) = common::init_rest_tester().unwrap(); + + let addr = tester.newaddress()?; + let txid: Txid = tester + .node_client() + .call("sendtoaddress", &[addr.to_string().into(), 0.1.into()]) + .unwrap(); + + // While a reader holds the mempool lock, the daemon fetch must still complete. If the fetch + // ran under the write lock it could not start until the reader is dropped. + let proxied_before = tester.proxied_rpc_count("ok"); + let reader = tester.mempool().read().unwrap(); + assert!(reader.lookup_txn(&txid).is_none()); + + let mempool = Arc::clone(tester.mempool()); + let daemon = Arc::clone(tester.daemon()); + let handle = thread::spawn(move || Mempool::ensure_tx(&mempool, &daemon, txid)); + + let deadline = Instant::now() + TIMEOUT; + while tester.proxied_rpc_count("ok") <= proxied_before { + assert!( + Instant::now() < deadline, + "daemon fetch did not complete while a mempool reader was active" + ); + thread::sleep(Duration::from_millis(10)); + } + drop(reader); + handle.join().unwrap().expect("ensure_tx failed"); + + assert!(tester.mempool().read().unwrap().lookup_txn(&txid).is_some()); + let res = get_json(rest_addr, &format!("/tx/{}", txid))?; + assert_eq!(res["status"]["confirmed"].as_bool(), Some(false)); + + // An already-indexed txid must return without the write lock or a daemon fetch, so it + // completes while another thread holds a reader. + let proxied_before = tester.proxied_rpc_count("ok"); + let reader = tester.mempool().read().unwrap(); + let (done_tx, done_rx) = mpsc::channel(); + let mempool = Arc::clone(tester.mempool()); + let daemon = Arc::clone(tester.daemon()); + thread::spawn(move || done_tx.send(Mempool::ensure_tx(&mempool, &daemon, txid))); + done_rx + .recv_timeout(TIMEOUT) + .expect("ensure_tx for an indexed txid blocked on the write lock")?; + drop(reader); + assert_eq!(tester.proxied_rpc_count("ok"), proxied_before); + + rest_handle.stop(); + Ok(()) +} + // Elements-only tests #[cfg(feature = "liquid")] @@ -1580,7 +1725,7 @@ fn test_rest_mempool_rbf_eviction() -> Result<()> { // Regression test for the mempool eviction panic ("missing mempool edge // for outpoint"): tx A and its RBF replacement B spend the same outpoint // and transiently coexist in the local mempool view when B is injected - // through the broadcast endpoint (add_by_txid) while A is still indexed. + // through the broadcast endpoint (ensure_tx) while A is still indexed. // B's add() clobbers A's `edges` entry; the next sync round evicts A and // must tolerate the missing/foreign edge instead of panicking. let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap(); @@ -1618,7 +1763,7 @@ fn test_rest_mempool_rbf_eviction() -> Result<()> { let b_hex = finalized["hex"].as_str().expect("finalized tx hex"); // Inject B through the electrs broadcast endpoint: the node accepts the - // replacement (evicting A node-side), and add_by_txid() indexes B locally + // replacement (evicting A node-side), and ensure_tx() indexes B locally // while A is still present - clobbering A's edges entry for the shared // outpoint. let broadcast_resp = ureq::post(&format!("http://{}/tx", rest_addr)).send(b_hex)?; diff --git a/tests/scan_limits.rs b/tests/scan_limits.rs new file mode 100644 index 000000000..83377118d --- /dev/null +++ b/tests/scan_limits.rs @@ -0,0 +1,394 @@ +use serde_json::Value; +use std::net; + +use electrs::chain::Txid; + +pub mod common; + +use common::Result; + +fn get_json(rest_addr: net::SocketAddr, path: &str) -> Result { + Ok(ureq::get(&format!("http://{}{}", rest_addr, path)) + .call()? + .into_body() + .read_json()?) +} + +fn get_allow_error( + rest_addr: net::SocketAddr, + path: &str, +) -> std::result::Result, ureq::Error> { + ureq::get(&format!("http://{}{}", rest_addr, path)) + .config() + .http_status_as_error(false) + .build() + .call() +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_utxo_limit_checked_against_final_state() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(5_000); + + let fund_txid = tester.send_multi(&addr1, amount, 101)?; + tester.mine()?; + + let resp = get_allow_error(rest_addr, &format!("/address/{}/utxo", addr1))?; + assert_eq!(resp.status(), 400); + assert_eq!( + resp.into_body().read_to_string()?, + "Too many unspent outputs" + ); + + let stats = get_json(rest_addr, &format!("/address/{}", addr1))?; + assert_eq!(stats["chain_stats"]["funded_txo_count"].as_u64(), Some(101)); + + tester.consolidate( + fund_txid, + 101, + amount, + bitcoin::Amount::from_sat(30_000), + &addr1, + )?; + tester.mine()?; + + let res = get_json(rest_addr, &format!("/address/{}/utxo", addr1))?; + let utxos = res.as_array().expect("array of utxos"); + assert_eq!(utxos.len(), 1); + + rest_handle.stop(); + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_utxo_limit_stays_until_consolidation_confirms() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(5_000); + + let fund_txid = tester.send_multi(&addr1, amount, 101)?; + tester.mine()?; + + let resp = get_allow_error(rest_addr, &format!("/address/{}/utxo", addr1))?; + assert_eq!(resp.status(), 400); + + tester.consolidate( + fund_txid, + 101, + amount, + bitcoin::Amount::from_sat(30_000), + &addr1, + )?; + + let resp = get_allow_error(rest_addr, &format!("/address/{}/utxo", addr1))?; + assert_eq!(resp.status(), 400); + + // Once mined, the limit is evaluated against the new confirmed state and clears. + tester.mine()?; + let res = get_json(rest_addr, &format!("/address/{}/utxo", addr1))?; + let utxos = res.as_array().expect("array of utxos"); + assert_eq!(utxos.len(), 1); + + rest_handle.stop(); + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_history_scan_limit_finishes_oversized_height() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 5).unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(5_000); + + tester.send_multi(&addr1, amount, 150)?; + tester.mine()?; + + for _ in 0..3 { + tester.send(&addr1, amount)?; + tester.mine()?; + } + + let mut stats = None; + for _ in 0..5 { + match get_json(rest_addr, &format!("/address/{}", addr1)) { + Ok(res) => { + stats = Some(res); + break; + } + Err(_) => continue, + } + } + let stats = stats.expect("scan should get past the oversized height and converge"); + assert_eq!(stats["chain_stats"]["funded_txo_count"].as_u64(), Some(153)); + + rest_handle.stop(); + Ok(()) +} + +#[test] +fn test_history_scan_limit_bounds_and_resumes() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 110).unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + + for _ in 0..130 { + tester.send(&addr1, amount)?; + tester.mine()?; + } + + let resp = get_allow_error(rest_addr, &format!("/address/{}", addr1))?; + assert_eq!(resp.status(), 400); + assert_eq!( + resp.into_body().read_to_string()?, + "Scripthash history too large to scan" + ); + + let mut stats = None; + for _ in 0..5 { + match get_json(rest_addr, &format!("/address/{}", addr1)) { + Ok(res) => { + stats = Some(res); + break; + } + Err(_) => continue, + } + } + let stats = stats.expect("scan should have converged within a few requests"); + assert_eq!(stats["chain_stats"]["funded_txo_count"].as_u64(), Some(130)); + + rest_handle.stop(); + Ok(()) +} + +#[test] +fn test_history_scan_limit_checkpoints_below_cache_threshold() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 5).unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + + for _ in 0..6 { + tester.send(&addr1, amount)?; + tester.mine()?; + } + + let resp = get_allow_error(rest_addr, &format!("/address/{}", addr1))?; + assert_eq!(resp.status(), 400); + + let mut stats = None; + for _ in 0..3 { + match get_json(rest_addr, &format!("/address/{}", addr1)) { + Ok(res) => { + stats = Some(res); + break; + } + Err(_) => continue, + } + } + let stats = + stats.expect("checkpoint below the cache threshold should still be saved and progress"); + assert_eq!(stats["chain_stats"]["funded_txo_count"].as_u64(), Some(6)); + + rest_handle.stop(); + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_utxo_scan_limit_checkpoints_below_cache_threshold() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 5).unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + + for _ in 0..6 { + tester.send_multi(&addr1, amount, 1)?; + tester.mine()?; + } + + let resp = get_allow_error(rest_addr, &format!("/address/{}/utxo", addr1))?; + assert_eq!(resp.status(), 400); + + let mut utxos = None; + for _ in 0..3 { + match get_json(rest_addr, &format!("/address/{}/utxo", addr1)) { + Ok(res) => { + utxos = Some(res); + break; + } + Err(_) => continue, + } + } + let utxos = + utxos.expect("checkpoint below the cache threshold should still be saved and progress"); + assert_eq!(utxos.as_array().expect("array of utxos").len(), 6); + + rest_handle.stop(); + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_history_scan_limit_errors_on_short_page() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 5).unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + + tester.send(&addr1, amount)?; + tester.mine()?; + + tester.send_multi(&addr1, amount, 20)?; + tester.mine()?; + + let resp = get_allow_error(rest_addr, &format!("/address/{}/txs/chain", addr1))?; + assert_eq!(resp.status(), 400); + assert_eq!( + resp.into_body().read_to_string()?, + "Scripthash history too large to scan" + ); + + rest_handle.stop(); + Ok(()) +} + +#[test] +fn test_history_cursor_unmatched_returns_empty() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = common::init_rest_tester().unwrap(); + + let addr2 = tester.newaddress()?; + let foreign_txid = tester.send(&addr2, bitcoin::Amount::from_sat(100_000))?; + tester.mine()?; + + let addr1 = tester.newaddress()?; + let mut addr1_txids: Vec = vec![]; + for _ in 0..3 { + addr1_txids.push(tester.send(&addr1, bitcoin::Amount::from_sat(100_000))?); + tester.mine()?; + } + + let cursor = addr1_txids[1]; + let res = get_json( + rest_addr, + &format!("/address/{}/txs/chain/{}", addr1, cursor), + )?; + let txs = res.as_array().expect("array of txs"); + assert!(txs + .iter() + .all(|tx| tx["txid"].as_str() != Some(cursor.to_string().as_str()))); + + let res = get_json( + rest_addr, + &format!("/address/{}/txs/chain/{}", addr1, foreign_txid), + )?; + assert_eq!(res.as_array().expect("array of txs").len(), 0); + + let never_confirmed = "22".repeat(32); + let res = get_json( + rest_addr, + &format!("/address/{}/txs/chain/{}", addr1, never_confirmed), + )?; + assert_eq!(res.as_array().expect("array of txs").len(), 0); + + rest_handle.stop(); + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_history_cursor_from_another_script_does_not_trip_the_scan_limit() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 5).unwrap(); + + let addr1 = tester.newaddress()?; + let addr2 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + + for _ in 0..4 { + tester.send_multi(&addr1, amount, 3)?; + tester.mine()?; + } + // Confirmed at a height where addr1 has no history rows. + let foreign_txid = tester.send_multi(&addr2, amount, 1)?; + tester.mine()?; + + let res = get_json( + rest_addr, + &format!("/address/{}/txs/chain/{}", addr1, foreign_txid), + )?; + assert_eq!(res.as_array().expect("array of txs").len(), 0); + + rest_handle.stop(); + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_precache_retries_until_the_scan_completes() -> Result<()> { + use electrs::new_index::precache; + + let (rest_handle, rest_addr, mut tester) = + common::init_rest_tester_with(|c| c.history_scan_limit = 5).unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + for _ in 0..12 { + tester.send(&addr1, amount)?; + tester.mine()?; + } + + let scripthash = precache::compute_script_hash(addr1.script_pubkey().as_bytes()); + precache::precache(tester.query().chain(), vec![scripthash]); + + let stats = get_json(rest_addr, &format!("/address/{}", addr1))?; + assert_eq!(stats["chain_stats"]["funded_txo_count"].as_u64(), Some(12)); + + rest_handle.stop(); + Ok(()) +} + +#[cfg(not(feature = "liquid"))] +#[test] +fn test_utxo_checkpoint_limit_bounds_the_saved_set() -> Result<()> { + let (rest_handle, rest_addr, mut tester) = common::init_rest_tester_with(|c| { + c.utxos_limit = 10; + c.utxos_checkpoint_limit = 20; + c.history_scan_limit = 128; + }) + .unwrap(); + + let addr1 = tester.newaddress()?; + let amount = bitcoin::Amount::from_sat(1_000); + for _ in 0..4 { + tester.send_multi(&addr1, amount, 64)?; + tester.mine()?; + } + + for _ in 0..4 { + let resp = get_allow_error(rest_addr, &format!("/address/{}/utxo", addr1))?; + assert_eq!(resp.status(), 400); + } + + let scripthash = + electrs::new_index::precache::compute_script_hash(addr1.script_pubkey().as_bytes()); + let key = [b"U".as_slice(), &scripthash[..]].concat(); + assert!( + tester.query().chain().store().cache_db().get(&key).is_none(), + "a utxo set above --utxos-checkpoint-limit must not be checkpointed" + ); + + rest_handle.stop(); + Ok(()) +}