mirror of
https://github.com/bitcoin/bitcoin.git
synced 2025-04-29 14:59:39 -04:00
fees: add sanity check to forecaster manager
- Only return fee rate forecast when 95th percentile txs of the last 6 blocks were seen in the node's mempool. - Also add functional test
This commit is contained in:
ismaelsadeeq
parent
491b4799e7
commit
fe5964a614
10 changed files with 331 additions and 116 deletions
|
|
@ -331,8 +331,10 @@ void Shutdown(NodeContext& node)
|
|||
}
|
||||
|
||||
// Drop transactions we were still watching, record fee estimations.
|
||||
// Unregister forecaster manager from validation interface.
|
||||
if (node.forecasterman) {
|
||||
node.forecasterman->GetBlockPolicyEstimator()->Flush();
|
||||
if (node.validation_signals) node.validation_signals->UnregisterValidationInterface(node.forecasterman.get());
|
||||
}
|
||||
|
||||
// FlushStateToDisk generates a ChainStateFlushed callback, which we should avoid missing
|
||||
|
|
@ -1721,12 +1723,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
|
|||
auto mempool_forecaster = std::make_shared<MemPoolForecaster>(node.mempool.get(), &(chainman.ActiveChainstate()));
|
||||
node.forecasterman->RegisterForecaster(mempool_forecaster);
|
||||
auto block_policy_estimator = std::make_shared<CBlockPolicyEstimator>(FeeestPath(args), read_stale_estimates);
|
||||
validation_signals.RegisterSharedValidationInterface(block_policy_estimator);
|
||||
// Flush block policy estimates to disk periodically
|
||||
scheduler.scheduleEvery([block_policy_estimator] { block_policy_estimator->FlushFeeEstimates(); }, FEE_FLUSH_INTERVAL);
|
||||
|
||||
// Register block policy estimator to forecaster manager
|
||||
node.forecasterman->RegisterForecaster(block_policy_estimator);
|
||||
|
||||
validation_signals.RegisterValidationInterface(node.forecasterman.get());
|
||||
}
|
||||
|
||||
assert(!node.peerman);
|
||||
|
|
|
|||
|
|
@ -578,21 +578,6 @@ CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath
|
|||
|
||||
CBlockPolicyEstimator::~CBlockPolicyEstimator() = default;
|
||||
|
||||
void CBlockPolicyEstimator::TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t /*unused*/)
|
||||
{
|
||||
processTransaction(tx);
|
||||
}
|
||||
|
||||
void CBlockPolicyEstimator::TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason /*unused*/, uint64_t /*unused*/)
|
||||
{
|
||||
removeTx(tx->GetHash());
|
||||
}
|
||||
|
||||
void CBlockPolicyEstimator::MempoolTransactionsRemovedForBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, unsigned int nBlockHeight)
|
||||
{
|
||||
processBlock(txs_removed_for_block, nBlockHeight);
|
||||
}
|
||||
|
||||
void CBlockPolicyEstimator::processTransaction(const NewMempoolTransactionInfo& tx)
|
||||
{
|
||||
LOCK(m_cs_fee_estimator);
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@
|
|||
#include <threadsafety.h>
|
||||
#include <uint256.h>
|
||||
#include <util/fs.h>
|
||||
#include <validationinterface.h>
|
||||
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
|
|
@ -150,7 +149,7 @@ struct FeeCalculation
|
|||
* a certain number of blocks. Every time a block is added to the best chain, this class records
|
||||
* stats on the transactions included in that block
|
||||
*/
|
||||
class CBlockPolicyEstimator : public Forecaster, public CValidationInterface
|
||||
class CBlockPolicyEstimator : public Forecaster
|
||||
{
|
||||
private:
|
||||
/** Track confirm delays up to 12 blocks for short horizon */
|
||||
|
|
@ -272,14 +271,6 @@ public:
|
|||
EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator);
|
||||
|
||||
protected:
|
||||
/** Overridden from CValidationInterface. */
|
||||
void TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t /*unused*/) override
|
||||
EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator);
|
||||
void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason /*unused*/, uint64_t /*unused*/) override
|
||||
EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator);
|
||||
void MempoolTransactionsRemovedForBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, unsigned int nBlockHeight) override
|
||||
EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator);
|
||||
|
||||
/** Overridden from Forecaster. */
|
||||
ForecastResult ForecastFeeRate(int target, bool conservative) const override
|
||||
EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator);
|
||||
|
|
|
|||
|
|
@ -2,21 +2,33 @@
|
|||
// Distributed under the MIT software license. See the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#include <chain.h>
|
||||
#include <kernel/mempool_entry.h>
|
||||
#include <kernel/mempool_removal_reason.h>
|
||||
#include <logging.h>
|
||||
#include <policy/fees/block_policy_estimator.h>
|
||||
#include <policy/fees/forecaster.h>
|
||||
#include <policy/fees/forecaster_man.h>
|
||||
#include <policy/fees/forecaster_util.h>
|
||||
#include <policy/policy.h>
|
||||
#include <validationinterface.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <utility>
|
||||
|
||||
FeeRateForecasterManager::FeeRateForecasterManager()
|
||||
{
|
||||
LOCK(cs);
|
||||
prev_mined_blocks.reserve(NUMBER_OF_BLOCKS);
|
||||
}
|
||||
|
||||
void FeeRateForecasterManager::RegisterForecaster(std::shared_ptr<Forecaster> forecaster)
|
||||
{
|
||||
LOCK(cs);
|
||||
forecasters.emplace(forecaster->GetForecastType(), forecaster);
|
||||
forecasters.emplace(forecaster->GetForecastType(), std::move(forecaster));
|
||||
}
|
||||
|
||||
// Retrieves the block policy estimator if available
|
||||
CBlockPolicyEstimator* FeeRateForecasterManager::GetBlockPolicyEstimator()
|
||||
{
|
||||
LOCK(cs);
|
||||
|
|
@ -32,16 +44,27 @@ std::pair<ForecastResult, std::vector<std::string>> FeeRateForecasterManager::Fo
|
|||
std::vector<std::string> err_messages;
|
||||
ForecastResult feerate_forecast;
|
||||
|
||||
for (const auto& forecaster : forecasters) {
|
||||
auto curr_forecast = forecaster.second->ForecastFeeRate(target, conservative);
|
||||
if (prev_mined_blocks.size() < NUMBER_OF_BLOCKS) {
|
||||
err_messages.emplace_back(strprintf("Insufficient data (at least %d blocks needs to be processed after startup)", NUMBER_OF_BLOCKS));
|
||||
return {feerate_forecast, err_messages};
|
||||
}
|
||||
|
||||
const bool mempool_healthy = IsMempoolHealthy();
|
||||
if (!mempool_healthy) {
|
||||
err_messages.emplace_back("Mempool is unreliable for fee rate forecasting.");
|
||||
}
|
||||
|
||||
for (const auto& [type, forecaster] : forecasters) {
|
||||
if (!mempool_healthy && type == ForecastType::MEMPOOL_FORECAST) continue;
|
||||
auto curr_forecast = forecaster->ForecastFeeRate(target, conservative);
|
||||
|
||||
if (curr_forecast.m_error.has_value()) {
|
||||
err_messages.emplace_back(
|
||||
strprintf("%s: %s", forecastTypeToString(forecaster.first), curr_forecast.m_error.value()));
|
||||
strprintf("%s: %s", forecastTypeToString(type), curr_forecast.m_error.value()));
|
||||
}
|
||||
|
||||
// Handle case where the block policy forecaster does not have enough data.
|
||||
if (forecaster.first == ForecastType::BLOCK_POLICY && curr_forecast.feerate.IsEmpty()) {
|
||||
if (type == ForecastType::BLOCK_POLICY && curr_forecast.feerate.IsEmpty()) {
|
||||
return {ForecastResult(), err_messages};
|
||||
}
|
||||
|
||||
|
|
@ -76,3 +99,103 @@ unsigned int FeeRateForecasterManager::MaximumTarget() const
|
|||
}
|
||||
return maximum_target;
|
||||
}
|
||||
|
||||
// Checks if the mempool is in a healthy state for fee rate forecasting
|
||||
bool FeeRateForecasterManager::IsMempoolHealthy() const
|
||||
{
|
||||
AssertLockHeld(cs);
|
||||
if (prev_mined_blocks.size() < NUMBER_OF_BLOCKS) return false;
|
||||
|
||||
for (size_t i = 1; i < prev_mined_blocks.size(); ++i) {
|
||||
const auto& current = prev_mined_blocks[i];
|
||||
const auto& previous = prev_mined_blocks[i - 1];
|
||||
|
||||
// TODO: Instead of just returning false; prevent all cases where blocks are disconnected (i.e reorg)
|
||||
// then assume this should not happen
|
||||
if (current.m_height != previous.m_height + 1) return false;
|
||||
|
||||
// TODO: Ensure validation interface sync before calling this, then throw if missing block weight.
|
||||
if (!current.m_block_weight) return false;
|
||||
|
||||
// Handle when the node has seen none of the block transactions then return false.
|
||||
if (current.m_removed_block_txs_weight == 0 && !current.empty) return false;
|
||||
// When the block is empty then continue to next iteration.
|
||||
if (current.m_removed_block_txs_weight == 0 && current.empty) continue;
|
||||
// Ensure the node has seen most of the transactions in the block in it's mempool.
|
||||
// TODO: handle cases where the block is just sparse
|
||||
if (current.m_removed_block_txs_weight / *current.m_block_weight < HEALTHY_BLOCK_PERCENTILE) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Calculates total block weight excluding the coinbase transaction
|
||||
size_t FeeRateForecasterManager::CalculateBlockWeight(const std::vector<CTransactionRef>& txs) const
|
||||
{
|
||||
AssertLockHeld(cs);
|
||||
return std::accumulate(txs.begin() + 1, txs.end(), 0u, [](size_t acc, const CTransactionRef& tx) {
|
||||
return acc + GetTransactionWeight(*tx);
|
||||
});
|
||||
}
|
||||
|
||||
// Updates mempool data for removed transactions during block connection
|
||||
void FeeRateForecasterManager::MempoolTransactionsRemovedForBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, unsigned int nBlockHeight)
|
||||
{
|
||||
GetBlockPolicyEstimator()->processBlock(txs_removed_for_block, nBlockHeight);
|
||||
LOCK(cs);
|
||||
size_t removed_weight = std::accumulate(txs_removed_for_block.begin(), txs_removed_for_block.end(), 0u,
|
||||
[](size_t acc, const auto& tx) { return acc + GetTransactionWeight(*tx.info.m_tx); });
|
||||
|
||||
// TODO: Store all block health data and then use only the latest NUMBER_OF_BLOCKS for mempool health assessment
|
||||
if (prev_mined_blocks.size() == NUMBER_OF_BLOCKS) {
|
||||
prev_mined_blocks.erase(prev_mined_blocks.begin());
|
||||
}
|
||||
prev_mined_blocks.emplace_back(nBlockHeight, static_cast<double>(removed_weight));
|
||||
}
|
||||
|
||||
// Updates previously mined block data when a new block is connected
|
||||
void FeeRateForecasterManager::BlockConnected(ChainstateRole /*unused*/, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex)
|
||||
{
|
||||
LOCK(cs);
|
||||
size_t height = pindex->nHeight;
|
||||
auto it = std::find_if(prev_mined_blocks.begin(), prev_mined_blocks.end(), [height](const auto& blk) {
|
||||
return blk.m_height == height;
|
||||
});
|
||||
|
||||
if (it != prev_mined_blocks.end()) {
|
||||
it->m_block_weight = static_cast<double>(CalculateBlockWeight(block->vtx));
|
||||
it->empty = it->m_block_weight.value() == 0.0;
|
||||
}
|
||||
}
|
||||
|
||||
void FeeRateForecasterManager::TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t /*unused*/)
|
||||
{
|
||||
GetBlockPolicyEstimator()->processTransaction(tx);
|
||||
}
|
||||
|
||||
void FeeRateForecasterManager::TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason /*unused*/, uint64_t /*unused*/)
|
||||
{
|
||||
GetBlockPolicyEstimator()->removeTx(tx->GetHash());
|
||||
}
|
||||
|
||||
std::vector<std::string> FeeRateForecasterManager::GetPreviouslyMinedBlockDataStr() const
|
||||
{
|
||||
LOCK(cs);
|
||||
std::vector<std::string> block_data_strings;
|
||||
block_data_strings.reserve(prev_mined_blocks.size() + 1);
|
||||
block_data_strings.emplace_back(strprintf("Tracked %d most-recent blocks.", prev_mined_blocks.size()));
|
||||
|
||||
for (const auto& block : prev_mined_blocks) {
|
||||
std::string block_str = strprintf("Block height: %d, %dWU txs were seen in the mempool", block.m_height, block.m_removed_block_txs_weight);
|
||||
|
||||
if (!block.m_block_weight.has_value()) {
|
||||
block_str += ", Block weight: Unknown.";
|
||||
} else {
|
||||
double mempool_percentage = (block.m_removed_block_txs_weight > 0) ? (block.m_removed_block_txs_weight / block.m_block_weight.value()) * 100 : 0;
|
||||
block_str += strprintf(", Block weight: %d WU (%.2f%% from mempool).", block.m_block_weight.value(), mempool_percentage);
|
||||
}
|
||||
block_data_strings.emplace_back(block_str);
|
||||
}
|
||||
return block_data_strings;
|
||||
}
|
||||
|
||||
FeeRateForecasterManager::~FeeRateForecasterManager() = default;
|
||||
|
|
|
|||
|
|
@ -5,50 +5,96 @@
|
|||
#ifndef BITCOIN_POLICY_FEES_FORECASTER_MAN_H
|
||||
#define BITCOIN_POLICY_FEES_FORECASTER_MAN_H
|
||||
|
||||
#include <primitives/transaction.h>
|
||||
#include <sync.h>
|
||||
#include <threadsafety.h>
|
||||
#include <validationinterface.h>
|
||||
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
class CBlockPolicyEstimator;
|
||||
class Forecaster;
|
||||
struct ForecastResult;
|
||||
|
||||
enum class ForecastType;
|
||||
enum class MemPoolRemovalReason;
|
||||
|
||||
struct RemovedMempoolTransactionInfo;
|
||||
struct NewMempoolTransactionInfo;
|
||||
struct ForecastResult;
|
||||
|
||||
// Constants for mempool sanity checks.
|
||||
constexpr size_t NUMBER_OF_BLOCKS = 6;
|
||||
constexpr double HEALTHY_BLOCK_PERCENTILE = 0.75;
|
||||
|
||||
/** \class FeeRateForecasterManager
|
||||
* Manages multiple fee rate forecasters.
|
||||
*/
|
||||
class FeeRateForecasterManager
|
||||
class FeeRateForecasterManager : public CValidationInterface
|
||||
{
|
||||
private:
|
||||
//! Map of all registered forecasters to their shared pointers.
|
||||
std::unordered_map<ForecastType, std::shared_ptr<Forecaster>> forecasters GUARDED_BY(cs);
|
||||
|
||||
mutable Mutex cs;
|
||||
mutable RecursiveMutex cs;
|
||||
|
||||
//! Structure to track the health of mined blocks.
|
||||
struct BlockData {
|
||||
size_t m_height; //!< Block height.
|
||||
bool empty{false}; //!< Whether the block is empty.
|
||||
double m_removed_block_txs_weight; //!< Removed transaction weight from the mempool.
|
||||
std::optional<double> m_block_weight; //!< Weight of the block.
|
||||
|
||||
BlockData(size_t height, double removed_block_txs_weight)
|
||||
: m_height(height), m_removed_block_txs_weight(removed_block_txs_weight) {}
|
||||
};
|
||||
|
||||
//! Tracks the statistics of previously mined blocks.
|
||||
std::vector<BlockData> prev_mined_blocks GUARDED_BY(cs);
|
||||
|
||||
//! Checks if recent mined blocks indicate a healthy mempool state.
|
||||
bool IsMempoolHealthy() const EXCLUSIVE_LOCKS_REQUIRED(cs);
|
||||
|
||||
//! Computes the total weight of transactions in a block.
|
||||
size_t CalculateBlockWeight(const std::vector<CTransactionRef>& txs) const EXCLUSIVE_LOCKS_REQUIRED(cs);
|
||||
|
||||
protected:
|
||||
/** Overridden from CValidationInterface. */
|
||||
void TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t /*unused*/) override
|
||||
EXCLUSIVE_LOCKS_REQUIRED(!cs);
|
||||
|
||||
void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason /*unused*/, uint64_t /*unused*/) override
|
||||
EXCLUSIVE_LOCKS_REQUIRED(!cs);
|
||||
|
||||
void MempoolTransactionsRemovedForBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, unsigned int nBlockHeight) override
|
||||
EXCLUSIVE_LOCKS_REQUIRED(!cs);
|
||||
|
||||
void BlockConnected(ChainstateRole /*unused*/, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override
|
||||
EXCLUSIVE_LOCKS_REQUIRED(!cs);
|
||||
|
||||
public:
|
||||
/**
|
||||
* Register a forecaster.
|
||||
* @param[in] forecaster shared pointer to a Forecaster instance.
|
||||
*/
|
||||
FeeRateForecasterManager() EXCLUSIVE_LOCKS_REQUIRED(!cs);
|
||||
virtual ~FeeRateForecasterManager();
|
||||
|
||||
//! Registers a new fee forecaster.
|
||||
void RegisterForecaster(std::shared_ptr<Forecaster> forecaster) EXCLUSIVE_LOCKS_REQUIRED(!cs);
|
||||
|
||||
/*
|
||||
* Return the pointer to block policy estimator.
|
||||
*/
|
||||
//! Returns a pointer to the block policy estimator.
|
||||
CBlockPolicyEstimator* GetBlockPolicyEstimator() EXCLUSIVE_LOCKS_REQUIRED(!cs);
|
||||
|
||||
//! Retrieves block data as a human-readable string.
|
||||
std::vector<std::string> GetPreviouslyMinedBlockDataStr() const EXCLUSIVE_LOCKS_REQUIRED(!cs);
|
||||
|
||||
/**
|
||||
* Get a fee rate forecast from all registered forecasters for a given confirmation target.
|
||||
* Estimates a fee rate using registered forecasters for a given confirmation target.
|
||||
*
|
||||
* Polls all registered forecasters and selects the lowest fee rate.
|
||||
* Iterates through all registered forecasters and selects the lowest viable fee estimate
|
||||
* with acceptable confidence.
|
||||
*
|
||||
* @param[in] target The target within which the transaction should be confirmed.
|
||||
* @param[in] conservative True if the package cannot be fee bumped later.
|
||||
* @return A pair consisting of the forecast result and a vector of forecaster names.
|
||||
* @param[in] target The target confirmation window for the transaction.
|
||||
* @return A pair containing the forecast result (if available) and any relevant error messages.
|
||||
*/
|
||||
std::pair<ForecastResult, std::vector<std::string>> ForecastFeeRateFromForecasters(int target, bool conservative) const
|
||||
EXCLUSIVE_LOCKS_REQUIRED(!cs);
|
||||
|
|
|
|||
|
|
@ -260,6 +260,7 @@ static const CRPCConvertParam vRPCConvertParams[] =
|
|||
{ "getorphantxs", 0, "verbosity" },
|
||||
{ "estimatesmartfee", 0, "conf_target" },
|
||||
{ "estimatesmartfee", 2, "block_policy_only" },
|
||||
{ "estimatesmartfee", 3, "verbose" },
|
||||
{ "estimaterawfee", 0, "conf_target" },
|
||||
{ "estimaterawfee", 1, "threshold" },
|
||||
{ "prioritisetransaction", 1, "dummy" },
|
||||
|
|
|
|||
|
|
@ -40,7 +40,8 @@ static RPCHelpMan estimatesmartfee()
|
|||
{"conf_target", RPCArg::Type::NUM, RPCArg::Optional::NO, "Confirmation target in blocks (1 - 1008)"},
|
||||
{"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"economical"}, "The fee estimate mode.\n"
|
||||
+ FeeModesDetail(std::string("default mode will be used"))},
|
||||
{"block_policy_only", RPCArg::Type::BOOL, RPCArg::Default{false}, "Whether to use block policy estimator only.\n"}
|
||||
{"block_policy_only", RPCArg::Type::BOOL, RPCArg::Default{false}, "Whether to use block policy estimator only.\n"},
|
||||
{"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "Whether the response should be verbose"},
|
||||
},
|
||||
RPCResult{
|
||||
RPCResult::Type::OBJ, "", "",
|
||||
|
|
@ -56,6 +57,9 @@ static RPCHelpMan estimatesmartfee()
|
|||
"fee estimation is able to return based on how long it has been running.\n"
|
||||
"An error is returned if not enough transactions and blocks\n"
|
||||
"have been observed to make an estimate for any number of blocks."},
|
||||
{RPCResult::Type::ARR, "stats", /*optional=*/true, strprintf("%d most-recent blocks stats (if there are any), only returned when verbose is true", NUMBER_OF_BLOCKS),
|
||||
{{RPCResult::Type::STR, "", "error"},
|
||||
}},
|
||||
}},
|
||||
RPCExamples{
|
||||
HelpExampleCli("estimatesmartfee", "6") +
|
||||
|
|
@ -120,6 +124,17 @@ static RPCHelpMan estimatesmartfee()
|
|||
errors.push_back(err);
|
||||
}
|
||||
result.pushKV("errors", std::move(errors));
|
||||
bool verbose = false;
|
||||
if (!request.params[3].isNull()) {
|
||||
verbose = request.params[3].get_bool();
|
||||
}
|
||||
if (verbose) {
|
||||
UniValue stats(UniValue::VARR);
|
||||
std::vector<std::string> verbose_stats = forecaster_man.GetPreviouslyMinedBlockDataStr();
|
||||
for (auto& stat : verbose_stats)
|
||||
stats.push_back(stat);
|
||||
result.pushKV("stats", stats);
|
||||
}
|
||||
return result;
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
#include <policy/fees/block_policy_estimator.h>
|
||||
#include <policy/fees/block_policy_estimator_args.h>
|
||||
#include <policy/fees/forecaster_man.h>
|
||||
#include <policy/policy.h>
|
||||
#include <test/util/txmempool.h>
|
||||
#include <txmempool.h>
|
||||
|
|
@ -15,13 +16,17 @@
|
|||
|
||||
#include <boost/test/unit_test.hpp>
|
||||
|
||||
#include <memory>
|
||||
|
||||
BOOST_FIXTURE_TEST_SUITE(policyestimator_tests, ChainTestingSetup)
|
||||
|
||||
BOOST_AUTO_TEST_CASE(BlockPolicyEstimates)
|
||||
{
|
||||
CBlockPolicyEstimator feeEst{FeeestPath(*m_node.args), DEFAULT_ACCEPT_STALE_FEE_ESTIMATES};
|
||||
auto forecasterman = std::make_unique<FeeRateForecasterManager>();
|
||||
auto feeEst = std::make_shared<CBlockPolicyEstimator>(FeeestPath(*m_node.args), DEFAULT_ACCEPT_STALE_FEE_ESTIMATES);
|
||||
forecasterman->RegisterForecaster(feeEst);
|
||||
CTxMemPool& mpool = *Assert(m_node.mempool);
|
||||
m_node.validation_signals->RegisterValidationInterface(&feeEst);
|
||||
m_node.validation_signals->RegisterValidationInterface(forecasterman.get());
|
||||
TestMemPoolEntryHelper entry;
|
||||
CAmount basefee(2000);
|
||||
CAmount deltaFee(100);
|
||||
|
|
@ -107,9 +112,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates)
|
|||
// At this point we should need to combine 3 buckets to get enough data points
|
||||
// So estimateFee(1) should fail and estimateFee(2) should return somewhere around
|
||||
// 9*baserate. estimateFee(2) %'s are 100,100,90 = average 97%
|
||||
BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0));
|
||||
BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() < 9*baseRate.GetFeePerK() + deltaFee);
|
||||
BOOST_CHECK(feeEst.estimateFee(2).GetFeePerK() > 9*baseRate.GetFeePerK() - deltaFee);
|
||||
BOOST_CHECK(feeEst->estimateFee(1) == CFeeRate(0));
|
||||
BOOST_CHECK(feeEst->estimateFee(2).GetFeePerK() < 9*baseRate.GetFeePerK() + deltaFee);
|
||||
BOOST_CHECK(feeEst->estimateFee(2).GetFeePerK() > 9*baseRate.GetFeePerK() - deltaFee);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -124,7 +129,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates)
|
|||
// Second highest feerate has 100% chance of being included by 2 blocks,
|
||||
// so estimateFee(2) should return 9*baseRate etc...
|
||||
for (int i = 1; i < 10;i++) {
|
||||
origFeeEst.push_back(feeEst.estimateFee(i).GetFeePerK());
|
||||
origFeeEst.push_back(feeEst->estimateFee(i).GetFeePerK());
|
||||
if (i > 2) { // Fee estimates should be monotonically decreasing
|
||||
BOOST_CHECK(origFeeEst[i-1] <= origFeeEst[i-2]);
|
||||
}
|
||||
|
|
@ -136,7 +141,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates)
|
|||
}
|
||||
// Fill out rest of the original estimates
|
||||
for (int i = 10; i <= 48; i++) {
|
||||
origFeeEst.push_back(feeEst.estimateFee(i).GetFeePerK());
|
||||
origFeeEst.push_back(feeEst->estimateFee(i).GetFeePerK());
|
||||
}
|
||||
|
||||
// Mine 50 more blocks with no transactions happening, estimates shouldn't change
|
||||
|
|
@ -149,10 +154,10 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates)
|
|||
// Wait for fee estimator to catch up
|
||||
m_node.validation_signals->SyncWithValidationInterfaceQueue();
|
||||
|
||||
BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0));
|
||||
BOOST_CHECK(feeEst->estimateFee(1) == CFeeRate(0));
|
||||
for (int i = 2; i < 10;i++) {
|
||||
BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee);
|
||||
BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee);
|
||||
BOOST_CHECK(feeEst->estimateFee(i).GetFeePerK() < origFeeEst[i-1] + deltaFee);
|
||||
BOOST_CHECK(feeEst->estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -192,7 +197,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates)
|
|||
m_node.validation_signals->SyncWithValidationInterfaceQueue();
|
||||
|
||||
for (int i = 1; i < 10;i++) {
|
||||
BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee);
|
||||
BOOST_CHECK(feeEst->estimateFee(i) == CFeeRate(0) || feeEst->estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee);
|
||||
}
|
||||
|
||||
// Mine all those transactions
|
||||
|
|
@ -215,9 +220,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates)
|
|||
// Wait for fee estimator to catch up
|
||||
m_node.validation_signals->SyncWithValidationInterfaceQueue();
|
||||
|
||||
BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0));
|
||||
BOOST_CHECK(feeEst->estimateFee(1) == CFeeRate(0));
|
||||
for (int i = 2; i < 10;i++) {
|
||||
BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee);
|
||||
BOOST_CHECK(feeEst->estimateFee(i) == CFeeRate(0) || feeEst->estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee);
|
||||
}
|
||||
|
||||
// Mine 400 more blocks where everything is mined every block
|
||||
|
|
@ -259,10 +264,11 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates)
|
|||
}
|
||||
// Wait for fee estimator to catch up
|
||||
m_node.validation_signals->SyncWithValidationInterfaceQueue();
|
||||
BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0));
|
||||
BOOST_CHECK(feeEst->estimateFee(1) == CFeeRate(0));
|
||||
for (int i = 2; i < 9; i++) { // At 9, the original estimate was already at the bottom (b/c scale = 2)
|
||||
BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee);
|
||||
BOOST_CHECK(feeEst->estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee);
|
||||
}
|
||||
m_node.validation_signals->UnregisterValidationInterface(forecasterman.get());
|
||||
}
|
||||
|
||||
BOOST_AUTO_TEST_SUITE_END()
|
||||
|
|
|
|||
|
|
@ -400,14 +400,13 @@ class EstimateFeeTest(BitcoinTestFramework):
|
|||
self.start_node(0,extra_args=["-acceptstalefeeestimates"])
|
||||
assert_equal(self.nodes[0].estimatesmartfee(1, "economical", True)["feerate"], fee_rate)
|
||||
|
||||
def clear_estimates(self):
|
||||
self.log.info("Restarting node with fresh estimation")
|
||||
self.stop_node(0)
|
||||
fee_dat = self.nodes[0].chain_path / "fee_estimates.dat"
|
||||
def clear_estimates(self, node_index=0):
|
||||
self.log.info(f"Restarting node{node_index} with fresh estimation")
|
||||
self.stop_node(node_index)
|
||||
fee_dat = self.nodes[node_index].chain_path / "fee_estimates.dat"
|
||||
os.remove(fee_dat)
|
||||
self.start_node(0)
|
||||
self.connect_nodes(0, 1)
|
||||
self.connect_nodes(0, 2)
|
||||
self.start_node(node_index)
|
||||
self.connect_all_nodes()
|
||||
self.sync_blocks()
|
||||
assert_equal(self.nodes[0].estimatesmartfee(1, "economical", True)["errors"], ["Insufficient data or no feerate found"])
|
||||
|
||||
|
|
@ -424,15 +423,28 @@ class EstimateFeeTest(BitcoinTestFramework):
|
|||
for _ in range(blocks):
|
||||
self.broadcast_and_mine(self.nodes[1], self.nodes[2], fee_rate, txs)
|
||||
|
||||
def send_transactions(self, utxos, fee_rate, target_vsize):
|
||||
|
||||
def send_transactions(self, utxos, fee_rate, target_vsize, node):
|
||||
for utxo in utxos:
|
||||
self.wallet.send_self_transfer(
|
||||
from_node=self.nodes[0],
|
||||
from_node=node,
|
||||
utxo_to_spend=utxo,
|
||||
fee_rate=fee_rate,
|
||||
target_vsize=target_vsize,
|
||||
)
|
||||
|
||||
def connect_all_nodes(self):
|
||||
self.connect_nodes(0, 1)
|
||||
self.connect_nodes(0, 2)
|
||||
self.connect_nodes(1, 2)
|
||||
|
||||
|
||||
def clear_mempool(self, node_index, extra_args=None):
|
||||
self.stop_node(node_index)
|
||||
node = self.nodes[node_index]
|
||||
os.remove(node.chain_path / "mempool.dat")
|
||||
self.restart_node(node_index, extra_args=extra_args)
|
||||
|
||||
|
||||
def test_estimation_modes(self):
|
||||
low_feerate = Decimal("0.001")
|
||||
|
|
@ -449,58 +461,92 @@ class EstimateFeeTest(BitcoinTestFramework):
|
|||
check_fee_estimates_btw_modes(self.nodes[0], high_feerate, low_feerate)
|
||||
|
||||
|
||||
def calculate_target_vsize(self, num_txs):
|
||||
"""Helper function to calculate the target vsize for transactions."""
|
||||
return int((DEFAULT_BLOCK_MAX_WEIGHT / WITNESS_SCALE_FACTOR - 8000) / num_txs)
|
||||
|
||||
|
||||
def mine_block_and_sync(self, node):
|
||||
"""Helper function to mine a single block and synchronize nodes."""
|
||||
self.generate(node, 1, sync_fun=self.no_op)
|
||||
self.sync_blocks()
|
||||
|
||||
|
||||
def test_estimatesmartfee_default(self):
|
||||
node0 = self.nodes[0]
|
||||
self.log.info("Ensure node0's mempool is empty at the start")
|
||||
node0, node1 = self.nodes[0], self.nodes[1]
|
||||
# Configure node0 to accept large transactions, allowing 10 to fill a block template
|
||||
node0_num_txs = 10
|
||||
node0_target_vsize = self.calculate_target_vsize(node0_num_txs)
|
||||
node0_extra_args = [f"-datacarriersize={node0_target_vsize}"]
|
||||
self.restart_node(0, node0_extra_args)
|
||||
|
||||
# Configure node1 to accept large transactions, allowing 20 to fill a block template
|
||||
node1_num_txs = 20
|
||||
node1_target_vsize = self.calculate_target_vsize(node1_num_txs)
|
||||
node1_extra_args = [f"-datacarriersize={node1_target_vsize}"]
|
||||
self.restart_node(1, node1_extra_args)
|
||||
|
||||
self.connect_all_nodes()
|
||||
self.log.info("Ensure the node mempool is empty at the start")
|
||||
assert_equal(node0.getmempoolinfo()['size'], 0)
|
||||
assert_equal(node1.getmempoolinfo()['size'], 0)
|
||||
|
||||
# Error messages for various scenarios
|
||||
errors = {
|
||||
"mempool_forecast": "Mempool Forecast: Forecaster unable a fee rate due to insufficient data",
|
||||
"block_policy": "Block Policy Estimator: Insufficient data or no feerate found",
|
||||
"lack_of_data": "Insufficient data (at least 6 blocks needs to be processed after startup)",
|
||||
"unreliable_mempool": "Mempool is unreliable for fee rate forecasting."
|
||||
}
|
||||
|
||||
self.log.info("Test estimatesmartfee after restart with empty mempool and no block policy estimator data")
|
||||
mempool_forecast_error = "Mempool Forecast: Forecaster unable a fee rate due to insufficient data"
|
||||
block_policy_error = "Block Policy Estimator: Insufficient data or no feerate found"
|
||||
estimate_after_restart = node0.estimatesmartfee(1)
|
||||
verify_estimate_response(estimate_after_restart, None, None, [block_policy_error])
|
||||
verify_estimate_response(node0.estimatesmartfee(1), None, None, [errors["lack_of_data"]])
|
||||
|
||||
self.log.info("Test estimatesmartfee after gathering sufficient block policy estimator data")
|
||||
# Generate high-feerate transactions and mine them over 6 blocks
|
||||
high_feerate, tx_count = Decimal("0.004"), 24
|
||||
self.log.info("Test estimatesmartfee after gathering sufficient block policy estimator data with empty mempool")
|
||||
high_feerate, tx_count = Decimal("0.00009"), 24
|
||||
self.setup_and_broadcast(high_feerate, blocks=6, txs=tx_count)
|
||||
estimate_from_block_policy = node0.estimatesmartfee(1)
|
||||
verify_estimate_response(estimate_from_block_policy, high_feerate, "Block Policy Estimator", [mempool_forecast_error])
|
||||
verify_estimate_response(node0.estimatesmartfee(1), high_feerate, "Block Policy Estimator", [errors["mempool_forecast"]])
|
||||
|
||||
self.log.info("Verify we return block policy estimator estimate when mempool provides higher estimate")
|
||||
# Add 10 large insane-feerate transactions enough to generate a block template
|
||||
num_txs = 10
|
||||
target_vsize = int((DEFAULT_BLOCK_MAX_WEIGHT / WITNESS_SCALE_FACTOR - 4000) / num_txs) # Account for coinbase space
|
||||
self.restart_node(0, extra_args=[f"-datacarriersize={target_vsize}"])
|
||||
utxos = [self.wallet.get_utxo(confirmed_only=True) for _ in range(num_txs)]
|
||||
insane_feerate = Decimal("0.01")
|
||||
self.send_transactions(utxos, insane_feerate, target_vsize)
|
||||
estimate_after_spike = node0.estimatesmartfee(1)
|
||||
verify_estimate_response(estimate_after_spike, high_feerate, "Block Policy Estimator", [])
|
||||
self.log.info("Test that estimate will be from mempool when it is lower than block policy")
|
||||
low_feerate = Decimal("0.00006")
|
||||
utxos = [self.wallet.get_utxo(confirmed_only=True) for _ in range(node0_num_txs)]
|
||||
self.send_transactions(utxos, low_feerate, node0_target_vsize, node0)
|
||||
verify_estimate_response(node0.estimatesmartfee(1), low_feerate, "Mempool Forecast", [])
|
||||
|
||||
self.log.info("Test caching of recent estimates")
|
||||
# Restart node with empty mempool, then broadcast low-feerate transactions
|
||||
# Check that estimate reflects the lower feerate even after higher-feerate transactions were recently broadcasted
|
||||
self.stop_node(0)
|
||||
os.remove(node0.chain_path / "mempool.dat")
|
||||
self.restart_node(0, extra_args=[f"-datacarriersize={target_vsize}"])
|
||||
|
||||
low_feerate = Decimal("0.00004")
|
||||
self.send_transactions(utxos, low_feerate, target_vsize)
|
||||
lower_estimate = node0.estimatesmartfee(1)
|
||||
verify_estimate_response(lower_estimate, low_feerate, "Mempool Forecast", [])
|
||||
|
||||
# Verify cache persists low-feerate estimate after broadcasting med-feerate transactions
|
||||
med_feerate = Decimal("0.002")
|
||||
self.send_transactions(utxos, med_feerate, target_vsize) # Double-spend UTXOs with medium feerate
|
||||
cached_estimate = node0.estimatesmartfee(1)
|
||||
verify_estimate_response(cached_estimate, low_feerate, "Mempool Forecast", [])
|
||||
med_feerate = Decimal("0.00008")
|
||||
self.send_transactions(utxos, med_feerate, node0_target_vsize, node0) # med-feerate RBF
|
||||
verify_estimate_response(node0.estimatesmartfee(1), low_feerate, "Mempool Forecast", [])
|
||||
|
||||
self.log.info("Test estimate refresh after cache expiration")
|
||||
current_timestamp = int(time.time())
|
||||
node0.setmocktime(current_timestamp + (MEMPOOL_FORECASTER_CACHE_LIFE + 1))
|
||||
new_estimate = node0.estimatesmartfee(1)
|
||||
verify_estimate_response(new_estimate, med_feerate, "Mempool Forecast", [])
|
||||
node0.setmocktime(int(time.time()) + (MEMPOOL_FORECASTER_CACHE_LIFE + 1))
|
||||
verify_estimate_response(node0.estimatesmartfee(1), med_feerate, "Mempool Forecast", [])
|
||||
|
||||
self.log.info("Test estimate on node1 which lacks node0's large transactions")
|
||||
verify_estimate_response(node1.estimatesmartfee(1), high_feerate, "Block Policy Estimator", [errors["mempool_forecast"]])
|
||||
|
||||
self.log.info("Test diverging node policy")
|
||||
self.connect_all_nodes()
|
||||
self.mine_block_and_sync(node0)
|
||||
self.wallet.rescan_utxos()
|
||||
feerate = Decimal("0.00007")
|
||||
new_utxos = [self.wallet.get_utxo(confirmed_only=True) for _ in range(node1_num_txs)]
|
||||
self.send_transactions(new_utxos, feerate, node1_target_vsize, node1)
|
||||
verify_estimate_response(node1.estimatesmartfee(1), high_feerate, "Block Policy Estimator", [errors["unreliable_mempool"]])
|
||||
|
||||
self.log.info("Test estimates when mempool is reliable")
|
||||
self.clear_mempool(0, node0_extra_args)
|
||||
self.clear_mempool(1, node1_extra_args)
|
||||
self.connect_all_nodes()
|
||||
self.setup_and_broadcast(high_feerate, blocks=6, txs=tx_count)
|
||||
self.send_transactions(new_utxos, feerate, node1_target_vsize, node1)
|
||||
verify_estimate_response(node1.estimatesmartfee(1), feerate, "Mempool Forecast", [])
|
||||
|
||||
self.log.info("Test that we provide block policy estimator estimate when it is lower")
|
||||
insane_feerate = Decimal("0.001")
|
||||
self.send_transactions(new_utxos, insane_feerate, node1_target_vsize, node1) # insane_feerate RBF
|
||||
node1.setmocktime(int(time.time()) + (MEMPOOL_FORECASTER_CACHE_LIFE + 1))
|
||||
verify_estimate_response(node1.estimatesmartfee(1), high_feerate, "Block Policy Estimator", [])
|
||||
|
||||
|
||||
def run_test(self):
|
||||
|
|
@ -517,9 +563,7 @@ class EstimateFeeTest(BitcoinTestFramework):
|
|||
# so the estimates would not be affected by the splitting transactions
|
||||
self.start_node(1)
|
||||
self.start_node(2)
|
||||
self.connect_nodes(1, 0)
|
||||
self.connect_nodes(0, 2)
|
||||
self.connect_nodes(2, 1)
|
||||
self.connect_all_nodes()
|
||||
self.sync_all()
|
||||
|
||||
self.log.info("Testing estimates with single transactions.")
|
||||
|
|
@ -550,6 +594,7 @@ class EstimateFeeTest(BitcoinTestFramework):
|
|||
self.test_estimation_modes()
|
||||
|
||||
self.clear_estimates()
|
||||
self.clear_estimates(1)
|
||||
self.log.info("Test estimatesmartfee default")
|
||||
self.test_estimatesmartfee_default()
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ class EstimateFeeTest(BitcoinTestFramework):
|
|||
assert_raises_rpc_error(-3, "JSON value of type string is not of expected type number", self.nodes[0].estimaterawfee, 1, 'foo')
|
||||
|
||||
# extra params
|
||||
assert_raises_rpc_error(-1, "estimatesmartfee", self.nodes[0].estimatesmartfee, 1, 'ECONOMICAL', True, 1)
|
||||
assert_raises_rpc_error(-1, "estimatesmartfee", self.nodes[0].estimatesmartfee, 1, 'ECONOMICAL', False, False, 1)
|
||||
assert_raises_rpc_error(-1, "estimaterawfee", self.nodes[0].estimaterawfee, 1, 1, 1)
|
||||
|
||||
# max value of 1008 per src/policy/fees.h
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue