Merge pull request #53 from distributed-lab/feature/unspend_utxo_indexer

UTXO Indexer

Author: ivanlele

utxo_indexer/indexer/Cargo.toml

@@ -3,4 +3,15 @@ name = "indexer"
 version = "0.1.0"
 edition = "2024"
 
+[[bin]]
+name = "indexer"
+path = "src/main.rs"
+
 [dependencies]
+anyhow = "1.0.100"
+bincode = "2.0.1"
+bitcoin = "0.32.7"
+byteorder = "1.5.0"
+clap = { version = "4.5.51", features = ["derive"] }
+hex = "0.4.3"
+rocksdb = "0.24.0"

utxo_indexer/indexer/src/bitcoin_primitives.rs

@@ -0,0 +1,125 @@
+use crate::bitcoin_serialization::{decompress_amount, read_var_int_u32, read_var_int_u64};
+
+use bitcoin::{
+    Txid, VarInt, consensus,
+    hashes::Hash,
+    opcodes::all::{OP_CHECKSIG, OP_DUP, OP_EQUALVERIFY, OP_HASH160},
+};
+
+#[allow(unused)]
+#[derive(Debug)]
+pub struct CoinKey {
+    pub txid: Txid,
+    pub vout: VarInt,
+}
+
+impl CoinKey {
+    const COIN_DELIMITER: u8 = 0x43; // 'C'
+
+    const MINIMUM_KEY_SIZE: usize = 1 + 32 + 1; // 1 byte prefix + 32 bytes txid + at least 1 byte vout
+
+    pub fn deserialize(data: &[u8]) -> Option<Self> {
+        if data.len() < Self::MINIMUM_KEY_SIZE {
+            return None;
+        }
+
+        let txid = Txid::from_slice(&data[1..33]).ok()?;
+        let vout = consensus::deserialize(&data[33..]).ok()?;
+
+        Some(Self { txid, vout })
+    }
+
+    #[allow(unused)]
+    pub fn serialize(&self) -> Vec<u8> {
+        let mut result = Vec::with_capacity(1 + 32 + self.vout.size());
+        result.push(Self::COIN_DELIMITER);
+        result.extend_from_slice(self.txid.as_raw_hash().as_byte_array());
+        result.extend_from_slice(&consensus::serialize(&self.vout));
+        result
+    }
+}
+
+#[allow(unused)]
+#[derive(Debug)]
+pub struct CoinValue {
+    pub height: u64,
+    pub is_coinbase: bool,
+    pub amount: u64,
+    pub script_pubkey: Vec<u8>,
+}
+
+impl CoinValue {
+    const P2PKH_COAT_OF_ARMS: u64 = 0x00;
+    const P2PKH_SCRIPT_LEN: usize = 25;
+    const PKH_SIZE: usize = 20;
+
+    pub fn deserialize(data: &[u8]) -> Option<Self> {
+        let mut cursor = 0;
+
+        let (code, consumed) = read_var_int_u32(data)?;
+
+        cursor += consumed;
+
+        let height = code >> 1;
+
+        let is_coinbase = (code & 1) == 1;
+
+        let (compressed_amount, consumed) = read_var_int_u32(&data[cursor..])?;
+        cursor += consumed;
+
+        let amount = decompress_amount(compressed_amount as u64);
+
+        let (script_len, consumed) = read_var_int_u64(&data[cursor..])?;
+
+        // TODO: implement the other script types
+        if Self::P2PKH_COAT_OF_ARMS != script_len {
+            return None;
+        }
+
+        cursor += consumed;
+
+        if data.len() < cursor + Self::PKH_SIZE {
+            return None;
+        }
+
+        let mut pkh = data[cursor..cursor + Self::PKH_SIZE].to_vec();
+
+        let mut script_pubkey = Vec::with_capacity(Self::P2PKH_SCRIPT_LEN);
+        script_pubkey.push(OP_DUP.to_u8());
+        script_pubkey.push(OP_HASH160.to_u8());
+        script_pubkey.push(20);
+        script_pubkey.append(&mut pkh);
+        script_pubkey.push(OP_EQUALVERIFY.to_u8());
+        script_pubkey.push(OP_CHECKSIG.to_u8());
+
+        Some(Self {
+            height: height as u64,
+            is_coinbase,
+            amount,
+            script_pubkey,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::str::FromStr;
+
+    use bitcoin::Txid;
+
+    #[test]
+    fn test_coin_key_deserialize() {
+        let key =
+            hex::decode("435a7b146bcde2a1f879c367fbbbac97a401aef899430182effb998d4ff1452a6d06")
+                .unwrap();
+
+        let tx_id =
+            Txid::from_str("6d2a45f14f8d99fbef82014399f8ae01a497acbbfb67c379f8a1e2cd6b147b5a")
+                .unwrap();
+
+        let deserialized = super::CoinKey::deserialize(&key).unwrap();
+
+        assert_eq!(deserialized.txid, tx_id);
+        assert_eq!(deserialized.vout, 6u32.into());
+    }
+}

utxo_indexer/indexer/src/bitcoin_serialization.rs

@@ -0,0 +1,82 @@
+pub fn deobfuscate(data: &[u8], obfuscation_key: &[u8]) -> Vec<u8> {
+    if obfuscation_key.is_empty() {
+        return data.to_vec();
+    }
+
+    data.iter()
+        .enumerate()
+        .map(|(i, &byte)| byte ^ obfuscation_key[i % obfuscation_key.len()])
+        .collect()
+}
+
+pub fn read_var_int_u64(data: &[u8]) -> Option<(u64, usize)> {
+    let mut result: u64 = 0;
+    let mut consumed = 0;
+
+    for byte in data {
+        consumed += 1;
+
+        if result > (u64::MAX >> 7) {
+            return None;
+        }
+
+        result = (result << 7) | u64::from(byte & 0x7F);
+        if byte & 0x80 != 0x00 {
+            if result == u64::MAX {
+                return None;
+            }
+
+            result += 1;
+        } else {
+            return Some((result, consumed));
+        }
+    }
+
+    None
+}
+
+pub fn read_var_int_u32(data: &[u8]) -> Option<(u32, usize)> {
+    let mut result: u32 = 0;
+    let mut consumed = 0;
+
+    for byte in data {
+        consumed += 1;
+
+        if result > (u32::MAX >> 7) {
+            return None;
+        }
+
+        result = (result << 7) | u32::from(byte & 0x7F);
+        if byte & 0x80 != 0x00 {
+            if result == u32::MAX {
+                return None;
+            }
+
+            result += 1;
+        } else {
+            return Some((result, consumed));
+        }
+    }
+
+    None
+}
+
+pub fn decompress_amount(x: u64) -> u64 {
+    if x == 0 {
+        return 0;
+    }
+
+    let mut x = x - 1;
+    let e = (x % 10) as u32;
+    x /= 10;
+
+    if e < 9 {
+        let d = (x % 9) + 1;
+        let n = x / 9;
+        let n_full = n * 10 + d;
+
+        n_full * 10u64.pow(e)
+    } else {
+        (x + 1) * 10u64.pow(9)
+    }
+}

utxo_indexer/indexer/src/cli.rs

@@ -0,0 +1,26 @@
+use clap::{Parser, Subcommand};
+
+#[derive(Parser)]
+#[command(version, about = "Builds UTXO index", long_about = None)]
+pub struct Cli {
+    #[command(subcommand)]
+    pub command: Commands,
+}
+
+#[derive(Subcommand)]
+pub enum Commands {
+    /// Index the chainstate and build the UTXO index
+    IndexChainstate {
+        /// Path to the chainstate LevelDB directory
+        #[arg(short, long)]
+        chainstate_path: String,
+        /// Path to output the UTXO index file
+        #[arg(short, long)]
+        output_path: String,
+    },
+    BuildMerkleRoot {
+        /// Path to the UTXO index file
+        #[arg(short, long)]
+        utxo_index_path: String,
+    },
+}

utxo_indexer/indexer/src/lib.rs

@@ -0,0 +1 @@
+

utxo_indexer/indexer/src/main.rs

@@ -0,0 +1,119 @@
+mod bitcoin_primitives;
+mod bitcoin_serialization;
+mod utils;
+
+mod cli;
+
+use anyhow::{Context, Result};
+use bitcoin::hashes::Hash;
+use clap::Parser;
+use rocksdb::{DB, IteratorMode, Options};
+
+use bitcoin::hashes::sha256;
+
+use crate::{
+    bitcoin_primitives::{CoinKey, CoinValue},
+    bitcoin_serialization::deobfuscate,
+    utils::{P2PKH_UTXO_SIZE, load_utxos, save_utxos},
+};
+
+const OBFUSCATION_KEY_DB_KEY: &[u8] = b"\x0e\x00obfuscate_key";
+
+fn main() -> Result<()> {
+    let cli = cli::Cli::parse();
+
+    match &cli.command {
+        cli::Commands::IndexChainstate {
+            chainstate_path,
+            output_path,
+        } => run_index_chainstate(chainstate_path.as_str(), output_path.as_str()),
+        cli::Commands::BuildMerkleRoot { utxo_index_path } => {
+            run_build_merkle_root(utxo_index_path.as_str())
+        }
+    }
+}
+
+fn run_index_chainstate(chainstate_path: &str, output_path: &str) -> Result<()> {
+    let mut opts = Options::default();
+    opts.set_compression_type(rocksdb::DBCompressionType::Snappy);
+    opts.create_if_missing(false);
+
+    let db = DB::open_for_read_only(&opts, chainstate_path, false)?;
+    let obfuscation_key_entry = db
+        .get(OBFUSCATION_KEY_DB_KEY)
+        .context("DB is not reachable")?
+        .context("obfuscation key is not present in DB")?;
+
+    println!(
+        "Opened chainstate at {:?}. Using obfuscation key: {}",
+        chainstate_path,
+        hex::encode(&obfuscation_key_entry[1..])
+    );
+    println!("Parsing UTXO entries...");
+
+    let iter = db.iterator(IteratorMode::Start);
+
+    let mut p2pkh_count = 0;
+
+    let mut utxos: Vec<[u8; P2PKH_UTXO_SIZE]> = Vec::with_capacity(100_000_000); // theoretical max amount of P2PKH UTXO in near future
+
+    for (i, item) in iter.enumerate() {
+        if i % 500_000 == 0 {
+            println!(
+                "Processed {} entries. Current P2PKH count: {}",
+                i, p2pkh_count
+            );
+        }
+
+        let (key, value) = item.context("DB is not reachable and iterable")?;
+
+        let _coin_key = match CoinKey::deserialize(&key) {
+            Some(ck) => ck,
+            None => continue,
+        };
+
+        let deobfuscated_value = deobfuscate(&value, &obfuscation_key_entry[1..]);
+
+        let coin_value = match CoinValue::deserialize(&deobfuscated_value) {
+            Some(cv) => cv,
+            None => continue,
+        };
+
+        p2pkh_count += 1;
+
+        let mut utxo = Vec::with_capacity(P2PKH_UTXO_SIZE);
+        utxo.extend_from_slice(&coin_value.amount.to_le_bytes());
+        utxo.extend_from_slice(&coin_value.script_pubkey);
+
+        utxos.push(utxo.try_into().expect("UTXO size should match"));
+    }
+
+    println!("Total P2PKH UTXO entries: {}", p2pkh_count);
+
+    save_utxos(&utxos, output_path)?;
+
+    println!("UTXO index saved to {}", output_path);
+
+    Ok(())
+}
+
+fn run_build_merkle_root(utxo_index_path: &str) -> Result<()> {
+    println!("Loading UTXO index from {}", utxo_index_path);
+
+    let utxos = load_utxos(utxo_index_path)?;
+
+    println!("Calculating Merkle root for {} UTXOs", utxos.len());
+
+    let mut merkle_tree_leaf_hashes: Vec<sha256::Hash> = Vec::with_capacity(utxos.len());
+    for utxo in utxos {
+        let leaf_hash = sha256::Hash::hash(&utxo);
+        merkle_tree_leaf_hashes.push(leaf_hash);
+    }
+
+    let root = bitcoin::merkle_tree::calculate_root(merkle_tree_leaf_hashes.into_iter())
+        .expect("UTXO set should not be empty");
+
+    println!("Merkle root: {}", hex::encode(root.as_byte_array()));
+
+    Ok(())
+}

utxo_indexer/indexer/src/utils.rs

@@ -0,0 +1,24 @@
+use std::{
+    fs::File,
+    io::{BufReader, BufWriter},
+};
+
+use anyhow::Result;
+
+pub const P2PKH_UTXO_SIZE: usize = 8 + 25; // 8 bytes for amount, 25 bytes for P2PKH scriptPubKey
+
+pub fn save_utxos(utxos: &Vec<[u8; P2PKH_UTXO_SIZE]>, path: &str) -> Result<()> {
+    let file = File::create(path)?;
+    let mut writer = BufWriter::new(file);
+    bincode::encode_into_std_write(utxos, &mut writer, bincode::config::standard())?;
+
+    Ok(())
+}
+
+pub fn load_utxos(path: &str) -> Result<Vec<[u8; P2PKH_UTXO_SIZE]>> {
+    let file = File::open(path)?;
+    let mut reader = BufReader::new(file);
+    let utxos = bincode::decode_from_std_read(&mut reader, bincode::config::standard())?;
+
+    Ok(utxos)
+}