Fail whole tx reads of pruned transactions.

include/bitcoin/database/impl/query/archive/chain_reader.ipp

@@ -183,6 +183,10 @@ typename CLASS::transaction::cptr CLASS::get_transaction(const tx_link& link,
         tx.locktime
     );
 
+    // A pruned tx (emptied input scripts) is short of its archived size.
+    if (ptr->serialized_size(witness) != (witness ? tx.heavy : tx.light))
+        return {};
+
     // TODO: store caches sizes so these could be forwarded.
     // Witness hash is not retained by the store.
     ptr->set_nominal_hash(std::move(tx.key));

include/bitcoin/database/impl/query/archive/wire_reader.ipp

@@ -214,7 +214,10 @@ bool CLASS::get_wire_tx(bytewriter& sink, const wire_accessors& ptrs,
     const auto ins_final = ins_begin + ins_count;
     const auto witnessed = witness && (tx.heavy != tx.light);
 
+    // Writers may encode, so the version write gives positions per byte.
+    const auto start = sink.get_write_position();
     sink.write_4_bytes_little_endian(tx.version);
+    const auto scale = (sink.get_write_position() - start) / sizeof(uint32_t);
 
     if (witnessed)
     {
@@ -246,7 +249,10 @@ bool CLASS::get_wire_tx(bytewriter& sink, const wire_accessors& ptrs,
     }
 
     sink.write_4_bytes_little_endian(tx.locktime);
-    return true;
+
+    // A pruned tx (emptied input scripts) is short of its archived size.
+    const auto size = witnessed ? tx.heavy : tx.light;
+    return sink.get_write_position() - start == scale * size;
 }
 
 TEMPLATE
@@ -301,8 +307,6 @@ data_chunk CLASS::get_wire_tx(const tx_link& link, bool witness) const NOEXCEPT
     if (!get_wire_tx(out, link, witness) || !out)
         return {};
 
-    // Resize in case tx is pruned.
-    data.resize(out.get_write_position());
     return data;
 }
 
@@ -324,8 +328,6 @@ data_chunk CLASS::get_wire_block(const header_link& link, bool witness,
     if (!get_wire_block(out, link, witness) || !out)
         return {};
 
-    // Resize in case tx is pruned.
-    data.resize(out.get_write_position());
     return data;
 }
 

include/bitcoin/database/tables/archives/transaction.hpp

@@ -134,7 +134,8 @@ struct transaction
     {
         inline bool from_data(reader& source) NOEXCEPT
         {
-            source.skip_bytes(skip_to_locktime);
+            light      = to_light(source.read_little_endian<bytes::integer, bytes::size>());
+            heavy      = source.read_little_endian<bytes::integer, bytes::size>();
             locktime   = source.read_little_endian<uint32_t>();
             version    = source.read_little_endian<uint32_t>();
             ins_count  = source.read_little_endian<ix::integer, ix::size>();
@@ -145,6 +146,8 @@ struct transaction
             return source;
         }
 
+        bytes::integer light{};
+        bytes::integer heavy{};
         uint32_t locktime{};
         uint32_t version{};
         ix::integer ins_count{};

include/bitcoin/database/tables/envelope.hpp

@@ -40,6 +40,12 @@ struct BCD_API envelope
 
     void set(const settings& database) NOEXCEPT;
 
+    /// Assign the stored values to configuration, returning the names of those
+    /// that differed. A differing top checkpoint, or provided filters, cannot
+    /// be assigned to configuration and are only returned.
+    system::string_list apply(system::settings& bitcoin, settings& database,
+        bool& limited, uint32_t& silent) const NOEXCEPT;
+
     bool from_data(reader& source) NOEXCEPT;
     bool to_data(flipper& sink) const NOEXCEPT;
     size_t serialized_size() const NOEXCEPT;

include/bitcoin/database/tables/schema.hpp

@@ -59,7 +59,7 @@ namespace schema {
 /// 4.0.13.0 Scanned and filtered block state flags, presilent batch tables.
 /// 4.0.14.0 Scan table head of scan starts, scan and tx counts at the latch.
 /// 4.0.15.0 Wtxid buckets and filter k in the envelope.
-constexpr std::array<uint32_t, 4> version{ 4, 0, 15, 0 };
+constexpr std::array<uint32_t, 4> version{ 4, 0, 16, 0 };
 
 /// Values.
 /// -----------------------------------------------------------------------

src/tables/envelope.cpp

@@ -54,6 +54,80 @@ void envelope::set(const settings& database) NOEXCEPT
         to_bool(database.filter_tx.buckets);
 }
 
+string_list envelope::apply(system::settings& bitcoin, settings& database,
+    bool& limited, uint32_t& silent) const NOEXCEPT
+{
+    string_list changed{};
+    const auto assign = [&](auto& configured, const auto& stored,
+        const std::string& name) NOEXCEPT
+    {
+        if (configured == stored)
+            return;
+
+        configured = stored;
+        changed.push_back(name);
+    };
+
+    assign(bitcoin.forks, forks, "forks");
+    assign(bitcoin.initial_subsidy_bitcoin, initial_subsidy_bitcoin, "initial_subsidy_bitcoin");
+    assign(bitcoin.subsidy_interval_blocks, subsidy_interval_blocks, "subsidy_interval_blocks");
+    assign(bitcoin.timestamp_limit_seconds, timestamp_limit_seconds, "timestamp_limit_seconds");
+    assign(bitcoin.retargeting_factor, retargeting_factor, "retargeting_factor");
+    assign(bitcoin.retargeting_interval_seconds, retargeting_interval_seconds, "retargeting_interval_seconds");
+    assign(bitcoin.block_spacing_seconds, block_spacing_seconds, "block_spacing_seconds");
+    assign(bitcoin.proof_of_work_limit, proof_of_work_limit, "proof_of_work_limit");
+    assign(bitcoin.first_version, first_version, "first_version");
+    assign(bitcoin.bip34_version, bip34_version, "bip34_version");
+    assign(bitcoin.bip66_version, bip66_version, "bip66_version");
+    assign(bitcoin.bip65_version, bip65_version, "bip65_version");
+    assign(bitcoin.bip9_version_bit0, bip9_version_bit0, "bip9_version_bit0");
+    assign(bitcoin.bip9_version_bit1, bip9_version_bit1, "bip9_version_bit1");
+    assign(bitcoin.bip9_version_bit2, bip9_version_bit2, "bip9_version_bit2");
+    assign(bitcoin.bip9_version_base, bip9_version_base, "bip9_version_base");
+    assign(bitcoin.bip16_activation_time, bip16_activation_time, "bip16_activation_time");
+    assign(bitcoin.bip34_activation_threshold, bip34_activation_threshold, "bip34_activation_threshold");
+    assign(bitcoin.bip34_enforcement_threshold, bip34_enforcement_threshold, "bip34_enforcement_threshold");
+    assign(bitcoin.bip34_activation_sample, bip34_activation_sample, "bip34_activation_sample");
+    assign(bitcoin.bip90_bip34_height, bip90_bip34_height, "bip90_bip34_height");
+    assign(bitcoin.bip90_bip65_height, bip90_bip65_height, "bip90_bip65_height");
+    assign(bitcoin.bip90_bip66_height, bip90_bip66_height, "bip90_bip66_height");
+    assign(bitcoin.bip30_reactivate_height, bip30_reactivate_height, "bip30_reactivate_height");
+    assign(bitcoin.bip30_deactivate_checkpoint, bip30_deactivate_checkpoint, "bip30_deactivate_checkpoint");
+    assign(bitcoin.bip9_bit0_active_checkpoint, bip9_bit0_active_checkpoint, "bip9_bit0_active_checkpoint");
+    assign(bitcoin.bip9_bit1_active_checkpoint, bip9_bit1_active_checkpoint, "bip9_bit1_active_checkpoint");
+    assign(bitcoin.bip9_bit2_active_checkpoint, bip9_bit2_active_checkpoint, "bip9_bit2_active_checkpoint");
+    assign(bitcoin.milestone, milestone, "milestone");
+
+    if (bitcoin.top_checkpoint() != top_checkpoint)
+        changed.push_back("top_checkpoint");
+
+    assign(database.interval_depth, interval_depth, "interval_depth");
+    assign(database.header.buckets, header_buckets, "header_buckets");
+    assign(database.ins.buckets, ins_buckets, "ins_buckets");
+    assign(database.outs.buckets, outs_buckets, "outs_buckets");
+    assign(database.tx.buckets, tx_buckets, "tx_buckets");
+    assign(database.duplicate.buckets, duplicate_buckets, "duplicate_buckets");
+    assign(database.pool.buckets, pool_buckets, "pool_buckets");
+    assign(database.wtxid.buckets, wtxid_buckets, "wtxid_buckets");
+
+    const auto filters = to_bool(database.filter_bk.buckets) &&
+        to_bool(database.filter_tx.buckets);
+
+    if (filters != provide_filters)
+    {
+        changed.push_back("provide_filters");
+        if (!provide_filters)
+        {
+            database.filter_bk.buckets = 0;
+            database.filter_tx.buckets = 0;
+        }
+    }
+
+    assign(limited, limited_blocks, "limited_blocks");
+    assign(silent, silent_start_height, "silent_start_height");
+    return changed;
+}
+
 envelope::envelope(const system::settings& bitcoin, const settings& database,
     bool limited, uint32_t silent) NOEXCEPT
   : schema(compiled),

test/query/archive/chain_reader.cpp

@@ -327,6 +327,44 @@ BOOST_AUTO_TEST_CASE(query_chain_reader__get_transactions__found__expected)
     BOOST_CHECK_EQUAL(query.get_transactions(2, false)->size(), 2u);
 }
 
+BOOST_AUTO_TEST_CASE(query_chain_reader__get_transaction__pruned__nullptr)
+{
+    settings settings{};
+    settings.path = TEST_DIRECTORY;
+    test::chunk_store store{ settings };
+    test::query_accessor query{ store };
+    BOOST_CHECK(!store.create(test::events_handler));
+    BOOST_CHECK(query.initialize(test::genesis));
+    BOOST_CHECK(query.set(test::block1.header(), test::context, {}, false));
+    BOOST_CHECK(query.set(test::block1, false, false, true));
+    BOOST_CHECK(!query.to_tx(test::block1.transactions_ptr()->front()->hash(false)).is_terminal());
+    BOOST_CHECK(!query.get_transaction(1, true));
+    BOOST_CHECK(!query.get_transaction(1, false));
+    BOOST_CHECK(!query.get_transactions(1, false));
+    BOOST_CHECK(!query.get_block(1, false));
+    BOOST_CHECK(query.get_header(1));
+}
+
+BOOST_AUTO_TEST_CASE(query_chain_reader__get_transaction__empty_input_script__expected)
+{
+    settings settings{};
+    settings.path = TEST_DIRECTORY;
+    test::chunk_store store{ settings };
+    test::query_accessor query{ store };
+    BOOST_CHECK(!store.create(test::events_handler));
+    BOOST_CHECK(query.initialize(test::genesis));
+
+    const system::chain::point point{ test::genesis.transactions_ptr()->front()->hash(false), 0 };
+    const system::chain::inputs inputs{ { point, system::chain::script{}, max_uint32 } };
+    const system::chain::outputs outputs{ { 42, system::chain::script{} } };
+    const system::chain::transaction tx{ 1, inputs, outputs, 0 };
+    BOOST_CHECK(query.set(tx));
+
+    const auto stored = query.get_transaction(1, true);
+    BOOST_REQUIRE(stored);
+    BOOST_CHECK(*stored == tx);
+}
+
 BOOST_AUTO_TEST_CASE(query_chain_reader__get_spenders__unspent_or_not_found__expected)
 {
     settings settings{};

test/query/archive/wire_reader.cpp

@@ -187,8 +187,68 @@ BOOST_AUTO_TEST_CASE(query_wire_reader__get_wire_tx__witness_false__expected)
     BOOST_CHECK(!store.close(test::events_handler));
 }
 
+BOOST_AUTO_TEST_CASE(query_wire_reader__get_wire_tx__pruned__empty)
+{
+    using namespace system;
+    database::settings settings{};
+    settings.path = TEST_DIRECTORY;
+    test::store_t store{ settings };
+    test::query_t query{ store };
+    BOOST_CHECK(!store.create(test::events_handler));
+    BOOST_CHECK(query.initialize(test::genesis));
+    BOOST_CHECK(query.set(test::block1.header(), database::context{ 0, 1, 0 }, {}, false));
+    BOOST_CHECK(query.set(test::block1, false, false, true));
+    BOOST_CHECK(!query.to_tx(test::block1.transactions_ptr()->front()->hash(false)).is_terminal());
+    BOOST_CHECK(query.get_wire_tx(1, true).empty());
+    BOOST_CHECK(query.get_wire_tx(1, false).empty());
+    BOOST_CHECK(!store.close(test::events_handler));
+}
+
+BOOST_AUTO_TEST_CASE(query_wire_reader__get_wire_tx__empty_input_script__expected)
+{
+    using namespace system;
+    database::settings settings{};
+    settings.path = TEST_DIRECTORY;
+    test::store_t store{ settings };
+    test::query_t query{ store };
+    BOOST_CHECK(!store.create(test::events_handler));
+    BOOST_CHECK(query.initialize(test::genesis));
+
+    const chain::point point{ test::genesis.transactions_ptr()->front()->hash(false), 0 };
+    const chain::inputs inputs{ { point, chain::script{}, max_uint32 } };
+    const chain::outputs outputs{ { 42, chain::script{} } };
+    const chain::transaction tx{ 1, inputs, outputs, 0 };
+    BOOST_CHECK(query.set(tx));
+    BOOST_CHECK_EQUAL(query.get_wire_tx(1, true), tx.to_data(true));
+    BOOST_CHECK_EQUAL(query.get_wire_tx(1, false), tx.to_data(false));
+    BOOST_CHECK(!store.close(test::events_handler));
+}
+
 // get_wire_block
 
+BOOST_AUTO_TEST_CASE(query_wire_reader__get_wire_block__pruned__empty)
+{
+    using namespace system;
+    database::settings settings{};
+    settings.path = TEST_DIRECTORY;
+    test::store_t store{ settings };
+    test::query_t query{ store };
+    BOOST_CHECK(!store.create(test::events_handler));
+    BOOST_CHECK(query.initialize(test::genesis));
+    BOOST_CHECK(query.set(test::block1.header(), database::context{ 0, 1, 0 }, {}, false));
+    BOOST_CHECK(query.set(test::block1, false, false, true));
+    BOOST_CHECK(query.get_wire_block(1, true).empty());
+    BOOST_CHECK(query.get_wire_block(1, false).empty());
+    BOOST_CHECK(query.get_wire_block(1, true, true).empty());
+
+    size_t size{};
+    BOOST_CHECK(query.get_block_size(size, 1, true));
+
+    std::string hex(two * size, '\0');
+    BOOST_CHECK(!query.get_wire_block<write::base16::fast>(hex, 1, true, false));
+    BOOST_CHECK(!store.close(test::events_handler));
+}
+
 BOOST_AUTO_TEST_CASE(query_wire_reader__get_wire_block__genesis_and_not__expected)
 {
     using namespace system;

test/tables/envelope.cpp

@@ -52,7 +52,7 @@ BOOST_AUTO_TEST_CASE(envelope__serialized_size__default__expected)
 BOOST_AUTO_TEST_CASE(envelope__to_data__default__expected_version)
 {
     const envelope instance{};
-    BOOST_REQUIRE_EQUAL(instance.schema, system::config::version(4, 0, 15, 0));
+    BOOST_REQUIRE_EQUAL(instance.schema, system::config::version(4, 0, 16, 0));
     BOOST_REQUIRE_EQUAL(to_chunk(instance).front(), schema::version.front());
 }
 
@@ -205,4 +205,51 @@ BOOST_AUTO_TEST_CASE(envelope__initialize__mainnet__expected)
     BOOST_REQUIRE_EQUAL(configuration.envelope.bip9_bit1_active_checkpoint.height(), 481824u);
 }
 
+BOOST_AUTO_TEST_CASE(envelope__apply__matching__unchanged)
+{
+    system::settings bitcoin{ chain::selection::mainnet };
+    database::settings configuration{ chain::selection::mainnet };
+    const envelope instance{ bitcoin, configuration, true, 42 };
+
+    auto limited = true;
+    auto silent = 42_u32;
+    BOOST_REQUIRE(instance.apply(bitcoin, configuration, limited, silent).empty());
+    BOOST_REQUIRE(limited);
+    BOOST_REQUIRE_EQUAL(silent, 42u);
+}
+
+BOOST_AUTO_TEST_CASE(envelope__apply__differing__assigned)
+{
+    system::settings bitcoin{ chain::selection::mainnet };
+    database::settings configuration{ chain::selection::mainnet };
+    const envelope instance{ bitcoin, configuration, true, 42 };
+
+    bitcoin.subsidy_interval_blocks = 1;
+    configuration.tx.buckets = 1;
+    auto limited = false;
+    auto silent = 7_u32;
+    const string_list expected{ "subsidy_interval_blocks", "tx_buckets", "limited_blocks", "silent_start_height" };
+    BOOST_REQUIRE(instance.apply(bitcoin, configuration, limited, silent) == expected);
+    BOOST_REQUIRE_EQUAL(bitcoin.subsidy_interval_blocks, instance.subsidy_interval_blocks);
+    BOOST_REQUIRE_EQUAL(configuration.tx.buckets, instance.tx_buckets);
+    BOOST_REQUIRE(limited);
+    BOOST_REQUIRE_EQUAL(silent, 42u);
+}
+
+BOOST_AUTO_TEST_CASE(envelope__apply__unprovided_filters__unbucketed)
+{
+    system::settings bitcoin{ chain::selection::mainnet };
+    database::settings configuration{ chain::selection::mainnet };
+    configuration.filter_bk.buckets = 0;
+    const envelope instance{ bitcoin, configuration, false, 42 };
+
+    configuration.filter_bk.buckets = 1;
+    auto limited = false;
+    auto silent = 42_u32;
+    const string_list expected{ "provide_filters" };
+    BOOST_REQUIRE(instance.apply(bitcoin, configuration, limited, silent) == expected);
+    BOOST_REQUIRE_EQUAL(configuration.filter_bk.buckets, 0u);
+    BOOST_REQUIRE_EQUAL(configuration.filter_tx.buckets, 0u);
+}
+
 BOOST_AUTO_TEST_SUITE_END()