Update fee estimator from pool queries.

include/bitcoin/node/chasers/chaser_transaction.hpp

@@ -62,6 +62,9 @@ class BCN_API chaser_transaction
     /// Recompute the pool context, closing the pool if not current.
     virtual void do_bump() NOEXCEPT;
 
+    /// Pool the unpooled txs of a reorganized block.
+    virtual void do_reorganized(header_t link) NOEXCEPT;
+
 private:
     static size_t to_rate(const system::chain::transaction& tx) NOEXCEPT;
 

include/bitcoin/node/error.hpp

@@ -121,6 +121,7 @@ enum error_t : uint8_t
     transaction4,
     transaction5,
     transaction6,
+    transaction7,
     estimates_initialize,
     estimates_push1,
     estimates_push2,

include/bitcoin/node/estimator.hpp

@@ -63,9 +63,9 @@ class BCN_API estimator
         size_t count=maximum_horizon) NOEXCEPT;
 
     /// Update accumulator (not thread safe).
-    bool push(const query& query) NOEXCEPT;
-    bool pop(const query& query) NOEXCEPT;
+    bool push(const query& query, const database::header_link& link) NOEXCEPT;
     bool pool(const query& query, const database::tx_link& link) NOEXCEPT;
+    bool pop(const query& query, const database::header_link& link) NOEXCEPT;
 
     /// Top height of accumulator (thread safe).
     size_t top_height() const NOEXCEPT;
@@ -167,6 +167,7 @@ class BCN_API estimator
     bool push(const rates& block) NOEXCEPT;
     bool pop(const rates& block) NOEXCEPT;
     bool pool(const rate& tx) NOEXCEPT;
+    void conflict(const rates& txs, bool push) NOEXCEPT;
     uint64_t compute(size_t target, double confidence,
         bool geometric=false) const NOEXCEPT;
 

src/chasers/chaser_estimate.cpp

@@ -232,7 +232,7 @@ void chaser_estimate::do_organized(header_t link) NOEXCEPT
         if (height.value <= estimator_->top_height())
             return;
 
-        if (!estimator_->push(query))
+        if (!estimator_->push(query, link))
             fault(error::estimates_push2);
     }
 }
@@ -256,7 +256,7 @@ void chaser_estimate::do_reorganized(header_t link) NOEXCEPT
         if (height.value > estimator_->top_height())
             return;
 
-        if (!estimator_->pop(query))
+        if (!estimator_->pop(query, link))
             fault(error::estimates_pop2);
     }
 }

src/chasers/chaser_transaction.cpp

@@ -60,11 +60,15 @@ bool chaser_transaction::handle_chase(const code&, event_value value) NOEXCEPT
     switch (to_chase(value))
     {
         case chase::organized:
-        case chase::reorganized:
         {
             POST(do_bump);
             break;
         }
+        case chase::reorganized:
+        {
+            POST(do_reorganized, to_payload<chase::reorganized>(value).link);
+            break;
+        }
         case chase::stop:
         {
             return false;
@@ -107,6 +111,49 @@ void chaser_transaction::do_bump() NOEXCEPT
     pooling_ = true;
 }
 
+// Pooled in block order under the block context, excluding any confirmed (by
+// another block).
+void chaser_transaction::do_reorganized(header_t link) NOEXCEPT
+{
+    BC_ASSERT(stranded());
+    do_bump();
+    if (!pooling_)
+        return;
+
+    auto& query = archive();
+    chain::context ctx{};
+    const auto txs = query.to_transactions(link);
+    if (txs.empty() || !query.get_context(ctx, link))
+    {
+        fault(error::transaction7);
+        return;
+    }
+
+    // The coinbase is never pooled.
+    for (auto it = std::next(txs.cbegin()); it != txs.cend(); ++it)
+    {
+        const database::tx_link fk{ *it };
+        if (query.is_pooled(fk) || query.is_confirmed_tx(fk))
+            continue;
+
+        const auto tx = query.get_transaction(fk, true);
+        if (!tx || !query.populate_with_metadata(*tx))
+        {
+            fault(error::transaction7);
+            return;
+        }
+
+        // Block txs are already silent indexed.
+        if (!query.set_pooled(fk, *tx, ctx))
+        {
+            fault(error::transaction2);
+            return;
+        }
+
+        notify(error::success, chases::transaction{ fk });
+    }
+}
+
 // methods
 // ----------------------------------------------------------------------------
 

src/error.cpp

@@ -111,6 +111,7 @@ DEFINE_ERROR_T_MESSAGE_MAP(error)
     { transaction4, "transaction4" },
     { transaction5, "transaction5" },
     { transaction6, "transaction6" },
+    { transaction7, "transaction7" },
     { estimates_initialize, "estimates_initialize" },
     { estimates_push1, "estimates_push1" },
     { estimates_push2, "estimates_push2" },

src/estimator.cpp

@@ -91,27 +91,37 @@ bool estimator::initialize(const std::atomic_bool& cancel, const query& query,
     rate_sets blocks{};
     const auto start = top - sub1(count);
     return query.get_branch_fees(cancel, blocks, start, count) &&
-        query.get_pool_fees(pooled) && initialize(blocks, pooled, top);
+        query.get_confirmable_fees(pooled) && initialize(blocks, pooled, top);
 }
 
-bool estimator::push(const query& query) NOEXCEPT
+bool estimator::push(const query& query,
+    const database::header_link& link) NOEXCEPT
 {
     if (is_add_overflow(top_height(), one))
         return false;
 
-    rates block{};
-    const auto link = query.to_confirmed(add1(top_height()));
-    return query.get_block_fees(block, link) && push(block);
+    rates block{}, conflicts{};
+    if (!query.get_block_fees(block, link) ||
+        !query.get_block_conflict_fees(conflicts, link) || !push(block))
+        return false;
+
+    conflict(conflicts, true);
+    return true;
 }
 
-bool estimator::pop(const query& query) NOEXCEPT
+bool estimator::pop(const query& query,
+    const database::header_link& link) NOEXCEPT
 {
     if (is_subtract_overflow(top_height(), one))
         return false;
 
-    rates block{};
-    const auto link = query.to_confirmed(sub1(top_height()));
-    return query.get_block_fees(block, link) && pop(block);
+    rates block{}, conflicts{};
+    if (!query.get_block_fees(block, link) ||
+        !query.get_block_conflict_fees(conflicts, link))
+        return false;
+
+    conflict(conflicts, false);
+    return pop(block);
 }
 
 bool estimator::pool(const query& query, const database::tx_link& link) NOEXCEPT
@@ -213,6 +223,17 @@ bool estimator::pool(const rate& tx) NOEXCEPT
     return true;
 }
 
+// Txs conflicted by a block leave pending when it is pushed, reenter on pop.
+void estimator::conflict(const rates& txs, bool push) NOEXCEPT
+{
+    for (const auto& tx: txs)
+    {
+        size_t bin{};
+        if (tx.height != max_size_t && to_bin(bin, tx))
+            pend(bin, tx.height, !push);
+    }
+}
+
 uint64_t estimator::compute(size_t target, double confidence,
     bool geometric) const NOEXCEPT
 {

src/protocols/protocol_transaction_out_60002.cpp

@@ -78,7 +78,6 @@ bool protocol_transaction_out_60002::handle_receive_memory_pool(
 // Outbound (inv).
 // ----------------------------------------------------------------------------
 
-// Each inventory is sent upon completion of the previous.
 void protocol_transaction_out_60002::send_memory_pool(const code& ec,
     const database::pool_link& cursor, const database::pool_link& end,
     const gate_t::ptr& gate) NOEXCEPT
@@ -94,7 +93,8 @@ void protocol_transaction_out_60002::send_memory_pool(const code& ec,
     while (items.size() < max_inventory && next != end)
     {
         const auto limit = max_inventory - items.size();
-        if (const auto result = query.get_pooled_txs(next, links, end, limit))
+        const auto result = query.get_confirmable_txs(next, links, end, limit);
+        if (result)
         {
             stop(result);
             return;

test/chasers/chaser_transaction.cpp

@@ -188,6 +188,21 @@ struct chaser_transaction_setup_fixture
         return false;
     }
 
+    bool pooled(const database::tx_link& link)
+    {
+        using namespace std::chrono;
+        const auto deadline = steady_clock::now() + seconds(10);
+        while (steady_clock::now() < deadline)
+        {
+            if (query_.is_pooled(link))
+                return true;
+
+            std::this_thread::sleep_for(milliseconds(10));
+        }
+
+        return false;
+    }
+
     // True if the command is received before the pong of a subsequent ping.
     bool received_before_pong(const std::string& command)
     {
@@ -510,6 +525,32 @@ BOOST_FIXTURE_TEST_CASE(chaser_transaction__organized__invalid_confirmed_top__su
     BOOST_REQUIRE_EQUAL(submit(package({ spend(parent_value) })).first, node::error::pooling_disabled);
 }
 
+// do_reorganized
+
+BOOST_FIXTURE_TEST_CASE(chaser_transaction__reorganized__unpooled_block_tx__pooled, chaser_transaction_pooling_setup_fixture)
+{
+    const system::settings bitcoin{ chain::selection::mainnet };
+    const auto& genesis = bitcoin.genesis_block;
+    const chain::input null{ chain::point{}, chain::script{}, max_uint32 };
+    const chain::transaction coinbase{ 1u, chain::inputs{ null }, chain::outputs{ { 42u, chain::script{} } }, 0u };
+    const auto tx = spend(parent_value);
+    const chain::header header{ 1u, genesis.hash(), null_hash, genesis.header().timestamp() + 1u, genesis.header().bits(), 0u };
+    const chain::block block{ header, { coinbase, *tx } };
+    BOOST_REQUIRE(query_.set(block, database::context{ 0, 1, 0 }, {}, false, true));
+
+    const auto link = query_.to_header(block.hash());
+    BOOST_REQUIRE(query_.push_candidate(link));
+    BOOST_REQUIRE(query_.push_confirmed(link, true));
+    BOOST_REQUIRE(query_.pop_confirmed());
+
+    const auto stored = query_.to_tx(tx->hash(false));
+    BOOST_REQUIRE(!query_.is_pooled(stored));
+
+    node_.notify(node::error::success, chases::reorganized{ link });
+    BOOST_REQUIRE(pooled(stored));
+    BOOST_REQUIRE(!query_.is_pooled(query_.to_tx(coinbase.hash(false))));
+}
+
 // submit_compact
 
 BOOST_FIXTURE_TEST_CASE(chaser_transaction__submit_compact__invalid_tx_link__integrity_suspended, chaser_transaction_header_setup_fixture)