diff --git a/.gitignore b/.gitignore index f77d3a49e11..54e4b9f6a66 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ target/ /guix-build-* /ci/scratch/ + +# Test data that includes .log files +!test/functional/data/coinstatsindex_v0/* diff --git a/src/index/base.h b/src/index/base.h index fbd9069a515..48480f4ac3e 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -74,9 +74,6 @@ private: /// with an empty datadir if, e.g., `-txindex=1` is specified. std::atomic m_synced{false}; - /// The last block in the chain that the index is in sync with. - std::atomic m_best_block_index{nullptr}; - std::thread m_thread_sync; CThreadInterrupt m_interrupt; @@ -94,11 +91,14 @@ private: bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip); virtual bool AllowPrune() const = 0; + virtual uint32_t GetVersion() const = 0; template void FatalErrorf(util::ConstevalFormatString fmt, const Args&... args); protected: + /// The last block in the chain that the index is in sync with. + std::atomic m_best_block_index{nullptr}; std::unique_ptr m_chain; Chainstate* m_chainstate{nullptr}; const std::string m_name; diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index ccb4845ef5e..43d49d78d0a 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -46,6 +46,7 @@ private: uint256 m_last_header{}; bool AllowPrune() const override { return true; } + uint32_t GetVersion() const override { return 0; } bool Write(const BlockFilter& filter, uint32_t block_height, const uint256& filter_header); diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index b5869416b93..ad179a0992f 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -23,6 +23,7 @@ using kernel::RemoveCoinHash; static constexpr uint8_t DB_BLOCK_HASH{'s'}; static constexpr uint8_t DB_BLOCK_HEIGHT{'t'}; static constexpr uint8_t DB_MUHASH{'M'}; +static constexpr uint8_t DB_VERSION{'V'}; namespace { @@ -31,15 +32,15 @@ struct DBVal { uint64_t transaction_output_count; uint64_t bogo_size; CAmount total_amount; - CAmount total_subsidy; + CAmount block_subsidy; CAmount total_unspendable_amount; - CAmount total_prevout_spent_amount; - CAmount total_new_outputs_ex_coinbase_amount; - CAmount total_coinbase_amount; - CAmount total_unspendables_genesis_block; - CAmount total_unspendables_bip30; - CAmount total_unspendables_scripts; - CAmount total_unspendables_unclaimed_rewards; + CAmount block_prevout_spent_amount; + CAmount block_new_outputs_ex_coinbase_amount; + CAmount block_coinbase_amount; + CAmount block_unspendables_genesis_block; + CAmount block_unspendables_bip30; + CAmount block_unspendables_scripts; + CAmount block_unspendables_unclaimed_rewards; SERIALIZE_METHODS(DBVal, obj) { @@ -47,15 +48,15 @@ struct DBVal { READWRITE(obj.transaction_output_count); READWRITE(obj.bogo_size); READWRITE(obj.total_amount); - READWRITE(obj.total_subsidy); + READWRITE(obj.block_subsidy); READWRITE(obj.total_unspendable_amount); - READWRITE(obj.total_prevout_spent_amount); - READWRITE(obj.total_new_outputs_ex_coinbase_amount); - READWRITE(obj.total_coinbase_amount); - READWRITE(obj.total_unspendables_genesis_block); - READWRITE(obj.total_unspendables_bip30); - READWRITE(obj.total_unspendables_scripts); - READWRITE(obj.total_unspendables_unclaimed_rewards); + READWRITE(obj.block_prevout_spent_amount); + READWRITE(obj.block_new_outputs_ex_coinbase_amount); + READWRITE(obj.block_coinbase_amount); + READWRITE(obj.block_unspendables_genesis_block); + READWRITE(obj.block_unspendables_bip30); + READWRITE(obj.block_unspendables_scripts); + READWRITE(obj.block_unspendables_unclaimed_rewards); } }; @@ -114,9 +115,18 @@ CoinStatsIndex::CoinStatsIndex(std::unique_ptr chain, size_t bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) { - CBlockUndo block_undo; - const CAmount block_subsidy{GetBlockSubsidy(block.height, Params().GetConsensus())}; - m_total_subsidy += block_subsidy; + CAmount block_unspendable{0}; + CBlockUndo block_undo{}; + + m_block_prevout_spent_amount = 0; + m_block_new_outputs_ex_coinbase_amount = 0; + m_block_coinbase_amount = 0; + m_block_subsidy = GetBlockSubsidy(block.height, Params().GetConsensus()); + + m_block_unspendables_genesis_block = 0; + m_block_unspendables_bip30 = 0; + m_block_unspendables_scripts = 0; + m_block_unspendables_unclaimed_rewards = 0; // Ignore genesis block if (block.height > 0) { @@ -151,8 +161,8 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) // Skip duplicate txid coinbase transactions (BIP30). if (IsBIP30Unspendable(*pindex) && tx->IsCoinBase()) { - m_total_unspendable_amount += block_subsidy; - m_total_unspendables_bip30 += block_subsidy; + block_unspendable += m_block_subsidy; + m_block_unspendables_bip30 += m_block_subsidy; continue; } @@ -163,17 +173,17 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) // Skip unspendable coins if (coin.out.scriptPubKey.IsUnspendable()) { - m_total_unspendable_amount += coin.out.nValue; - m_total_unspendables_scripts += coin.out.nValue; + block_unspendable += coin.out.nValue; + m_block_unspendables_scripts += coin.out.nValue; continue; } ApplyCoinHash(m_muhash, outpoint, coin); if (tx->IsCoinBase()) { - m_total_coinbase_amount += coin.out.nValue; + m_block_coinbase_amount += coin.out.nValue; } else { - m_total_new_outputs_ex_coinbase_amount += coin.out.nValue; + m_block_new_outputs_ex_coinbase_amount += coin.out.nValue; } ++m_transaction_output_count; @@ -191,7 +201,7 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) RemoveCoinHash(m_muhash, outpoint, coin); - m_total_prevout_spent_amount += coin.out.nValue; + m_block_prevout_spent_amount += coin.out.nValue; --m_transaction_output_count; m_total_amount -= coin.out.nValue; @@ -201,32 +211,33 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) } } else { // genesis block - m_total_unspendable_amount += block_subsidy; - m_total_unspendables_genesis_block += block_subsidy; + block_unspendable += m_block_subsidy; + m_block_unspendables_genesis_block += m_block_subsidy; } // If spent prevouts + block subsidy are still a higher amount than // new outputs + coinbase + current unspendable amount this means // the miner did not claim the full block reward. Unclaimed block // rewards are also unspendable. - const CAmount unclaimed_rewards{(m_total_prevout_spent_amount + m_total_subsidy) - (m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount)}; - m_total_unspendable_amount += unclaimed_rewards; - m_total_unspendables_unclaimed_rewards += unclaimed_rewards; + m_block_unspendables_unclaimed_rewards = (m_block_prevout_spent_amount + m_block_subsidy) - (m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + block_unspendable); + m_total_unspendable_amount += (m_block_unspendables_unclaimed_rewards + block_unspendable); std::pair value; value.first = block.hash; value.second.transaction_output_count = m_transaction_output_count; value.second.bogo_size = m_bogo_size; value.second.total_amount = m_total_amount; - value.second.total_subsidy = m_total_subsidy; value.second.total_unspendable_amount = m_total_unspendable_amount; - value.second.total_prevout_spent_amount = m_total_prevout_spent_amount; - value.second.total_new_outputs_ex_coinbase_amount = m_total_new_outputs_ex_coinbase_amount; - value.second.total_coinbase_amount = m_total_coinbase_amount; - value.second.total_unspendables_genesis_block = m_total_unspendables_genesis_block; - value.second.total_unspendables_bip30 = m_total_unspendables_bip30; - value.second.total_unspendables_scripts = m_total_unspendables_scripts; - value.second.total_unspendables_unclaimed_rewards = m_total_unspendables_unclaimed_rewards; + + value.second.block_subsidy = m_block_subsidy; + value.second.block_prevout_spent_amount = m_block_prevout_spent_amount; + value.second.block_new_outputs_ex_coinbase_amount = m_block_new_outputs_ex_coinbase_amount; + value.second.block_coinbase_amount = m_block_coinbase_amount; + + value.second.block_unspendables_genesis_block = m_block_unspendables_genesis_block; + value.second.block_unspendables_bip30 = m_block_unspendables_bip30; + value.second.block_unspendables_scripts = m_block_unspendables_scripts; + value.second.block_unspendables_unclaimed_rewards = m_block_unspendables_unclaimed_rewards; uint256 out; m_muhash.Finalize(out); @@ -337,21 +348,51 @@ std::optional CoinStatsIndex::LookUpStats(const CBlockIndex& block_ stats.nTransactionOutputs = entry.transaction_output_count; stats.nBogoSize = entry.bogo_size; stats.total_amount = entry.total_amount; - stats.total_subsidy = entry.total_subsidy; stats.total_unspendable_amount = entry.total_unspendable_amount; - stats.total_prevout_spent_amount = entry.total_prevout_spent_amount; - stats.total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount; - stats.total_coinbase_amount = entry.total_coinbase_amount; - stats.total_unspendables_genesis_block = entry.total_unspendables_genesis_block; - stats.total_unspendables_bip30 = entry.total_unspendables_bip30; - stats.total_unspendables_scripts = entry.total_unspendables_scripts; - stats.total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards; + + stats.block_subsidy = entry.block_subsidy; + stats.block_prevout_spent_amount = entry.block_prevout_spent_amount; + stats.block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount; + stats.block_coinbase_amount = entry.block_coinbase_amount; + + stats.block_unspendables_genesis_block = entry.block_unspendables_genesis_block; + stats.block_unspendables_bip30 = entry.block_unspendables_bip30; + stats.block_unspendables_scripts = entry.block_unspendables_scripts; + stats.block_unspendables_unclaimed_rewards = entry.block_unspendables_unclaimed_rewards; return stats; } bool CoinStatsIndex::CustomInit(const std::optional& block) { + uint32_t code_version{GetVersion()}; + uint32_t db_version{0}; + // We are starting the index for the first time and write version first so + // we don't run into the version check later. + if (!block.has_value() && !m_db->Exists(DB_VERSION)) { + m_db->Write(DB_VERSION, code_version); + db_version = code_version; + } + + // If we can't read a version this means the index has never been updated + // and needs to be reset now. Otherwise request a reset if we have a + // version mismatch. + if (m_db->Exists(DB_VERSION)) { + m_db->Read(DB_VERSION, db_version); + } + if (db_version == 0 && code_version == 1) { + // Attempt to migrate coinstatsindex without the need to reindex + if (!MigrateToV1()) { + LogError("%s migration: Error while migrating to v1. In order to rebuild the index, remove the indexes/coinstats directory in your datadir\n", + GetName()); + return false; + }; + } else if (db_version != code_version) { + LogError("%s version mismatch: expected %s but %s was found. In order to rebuild the index, remove the indexes/coinstats directory in your datadir\n", + GetName(), code_version, db_version); + return false; + } + if (!m_db->Read(DB_MUHASH, m_muhash)) { // Check that the cause of the read failure is that the key does not // exist. Any other errors indicate database corruption or a disk @@ -382,15 +423,17 @@ bool CoinStatsIndex::CustomInit(const std::optional& block m_transaction_output_count = entry.transaction_output_count; m_bogo_size = entry.bogo_size; m_total_amount = entry.total_amount; - m_total_subsidy = entry.total_subsidy; m_total_unspendable_amount = entry.total_unspendable_amount; - m_total_prevout_spent_amount = entry.total_prevout_spent_amount; - m_total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount; - m_total_coinbase_amount = entry.total_coinbase_amount; - m_total_unspendables_genesis_block = entry.total_unspendables_genesis_block; - m_total_unspendables_bip30 = entry.total_unspendables_bip30; - m_total_unspendables_scripts = entry.total_unspendables_scripts; - m_total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards; + + m_block_subsidy = entry.block_subsidy; + m_block_prevout_spent_amount = entry.block_prevout_spent_amount; + m_block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount; + m_block_coinbase_amount = entry.block_coinbase_amount; + + m_block_unspendables_genesis_block = entry.block_unspendables_genesis_block; + m_block_unspendables_bip30 = entry.block_unspendables_bip30; + m_block_unspendables_scripts = entry.block_unspendables_scripts; + m_block_unspendables_unclaimed_rewards = entry.block_unspendables_unclaimed_rewards; } return true; @@ -410,9 +453,6 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex CBlockUndo block_undo; std::pair read_out; - const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())}; - m_total_subsidy -= block_subsidy; - // Ignore genesis block if (pindex->nHeight > 0) { if (!m_chainstate->m_blockman.ReadBlockUndo(block_undo, *pindex)) { @@ -448,18 +488,11 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex // Skip unspendable coins if (coin.out.scriptPubKey.IsUnspendable()) { m_total_unspendable_amount -= coin.out.nValue; - m_total_unspendables_scripts -= coin.out.nValue; continue; } RemoveCoinHash(m_muhash, outpoint, coin); - if (tx->IsCoinBase()) { - m_total_coinbase_amount -= coin.out.nValue; - } else { - m_total_new_outputs_ex_coinbase_amount -= coin.out.nValue; - } - --m_transaction_output_count; m_total_amount -= coin.out.nValue; m_bogo_size -= GetBogoSize(coin.out.scriptPubKey); @@ -475,8 +508,6 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex ApplyCoinHash(m_muhash, outpoint, coin); - m_total_prevout_spent_amount -= coin.out.nValue; - m_transaction_output_count++; m_total_amount += coin.out.nValue; m_bogo_size += GetBogoSize(coin.out.scriptPubKey); @@ -484,27 +515,81 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex } } - const CAmount unclaimed_rewards{(m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount) - (m_total_prevout_spent_amount + m_total_subsidy)}; - m_total_unspendable_amount -= unclaimed_rewards; - m_total_unspendables_unclaimed_rewards -= unclaimed_rewards; - - // Check that the rolled back internal values are consistent with the DB read out + // Check that the rolled back internal values are consistent with the DB + // read out where possible, i.e. when total historical values are tracked. + // Otherwise just read the values from the index entry. uint256 out; m_muhash.Finalize(out); Assert(read_out.second.muhash == out); - Assert(m_transaction_output_count == read_out.second.transaction_output_count); Assert(m_total_amount == read_out.second.total_amount); - Assert(m_bogo_size == read_out.second.bogo_size); - Assert(m_total_subsidy == read_out.second.total_subsidy); + m_total_unspendable_amount -= m_block_unspendables_unclaimed_rewards; Assert(m_total_unspendable_amount == read_out.second.total_unspendable_amount); - Assert(m_total_prevout_spent_amount == read_out.second.total_prevout_spent_amount); - Assert(m_total_new_outputs_ex_coinbase_amount == read_out.second.total_new_outputs_ex_coinbase_amount); - Assert(m_total_coinbase_amount == read_out.second.total_coinbase_amount); - Assert(m_total_unspendables_genesis_block == read_out.second.total_unspendables_genesis_block); - Assert(m_total_unspendables_bip30 == read_out.second.total_unspendables_bip30); - Assert(m_total_unspendables_scripts == read_out.second.total_unspendables_scripts); - Assert(m_total_unspendables_unclaimed_rewards == read_out.second.total_unspendables_unclaimed_rewards); + Assert(m_transaction_output_count == read_out.second.transaction_output_count); + Assert(m_bogo_size == read_out.second.bogo_size); + + m_block_subsidy = read_out.second.block_subsidy; + m_block_prevout_spent_amount = read_out.second.block_prevout_spent_amount; + m_block_new_outputs_ex_coinbase_amount = read_out.second.block_new_outputs_ex_coinbase_amount; + m_block_coinbase_amount = read_out.second.block_coinbase_amount; + + m_block_unspendables_genesis_block = read_out.second.block_unspendables_genesis_block; + m_block_unspendables_bip30 = read_out.second.block_unspendables_bip30; + m_block_unspendables_scripts = read_out.second.block_unspendables_scripts; + m_block_unspendables_unclaimed_rewards = read_out.second.block_unspendables_unclaimed_rewards; return true; } + +bool CoinStatsIndex::MigrateToV1() { + LogInfo("Migrating coinstatsindex to new format (v1). This might take a few minutes.\n"); + CDBBatch batch(*m_db); + DBVal entry; + DBVal entry_prev; + const CBlockIndex *pindex{m_best_block_index}; + if (!LookUpOne(*m_db, {pindex->GetBlockHash(), pindex->nHeight}, entry)) { + return false; + } + + // Loop backwards until we hit the genesis block which doesn't need to be updated + while (pindex->pprev) { + if (pindex->nHeight % 10000 == 0) LogInfo("Migrating block at height %i\n", pindex->nHeight); + // Load Previous entry + if (!LookUpOne(*m_db, {pindex->pprev->GetBlockHash(), pindex->pprev->nHeight}, entry_prev)) { + LogError("Coinstatsindex is corrupted at height %i\n", pindex->pprev->nHeight); + return false; + } + // Combine entries + if (entry.block_subsidy < entry_prev.block_subsidy + || entry.block_prevout_spent_amount < entry_prev.block_prevout_spent_amount + || entry.block_prevout_spent_amount < entry_prev.block_prevout_spent_amount + || entry.block_new_outputs_ex_coinbase_amount < entry_prev.block_new_outputs_ex_coinbase_amount + || entry.block_coinbase_amount < entry_prev.block_coinbase_amount + || entry.block_unspendables_genesis_block < entry_prev.block_unspendables_genesis_block + || entry.block_unspendables_bip30 < entry_prev.block_unspendables_bip30 + || entry.block_unspendables_scripts < entry_prev.block_unspendables_scripts + || entry.block_unspendables_unclaimed_rewards < entry_prev.block_unspendables_unclaimed_rewards + ) { + LogError("Coinstatsindex is corrupted at height %i\n", pindex->nHeight); + return false; + } + entry.block_subsidy = entry.block_subsidy - entry_prev.block_subsidy; + entry.block_prevout_spent_amount = entry.block_prevout_spent_amount - entry_prev.block_prevout_spent_amount; + entry.block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount - entry_prev.block_new_outputs_ex_coinbase_amount; + entry.block_coinbase_amount = entry.block_coinbase_amount - entry_prev.block_coinbase_amount; + entry.block_unspendables_genesis_block = entry.block_unspendables_genesis_block - entry_prev.block_unspendables_genesis_block; + entry.block_unspendables_bip30 = entry.block_unspendables_bip30 - entry_prev.block_unspendables_bip30; + entry.block_unspendables_scripts = entry.block_unspendables_scripts - entry_prev.block_unspendables_scripts; + entry.block_unspendables_unclaimed_rewards = entry.block_unspendables_unclaimed_rewards - entry_prev.block_unspendables_unclaimed_rewards; + std::pair result; + result.first = pindex->GetBlockHash(); + result.second = entry; + batch.Write(DBHeightKey(pindex->nHeight), result); + pindex = pindex->pprev; + entry = entry_prev; + } + batch.Write(DB_VERSION, 1); + if (!m_db->WriteBatch(batch)) return false; + LogInfo("Migration succeeded\n"); + return true; +} diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h index 885b9e0a860..e8799181512 100644 --- a/src/index/coinstatsindex.h +++ b/src/index/coinstatsindex.h @@ -28,19 +28,23 @@ private: uint64_t m_transaction_output_count{0}; uint64_t m_bogo_size{0}; CAmount m_total_amount{0}; - CAmount m_total_subsidy{0}; CAmount m_total_unspendable_amount{0}; - CAmount m_total_prevout_spent_amount{0}; - CAmount m_total_new_outputs_ex_coinbase_amount{0}; - CAmount m_total_coinbase_amount{0}; - CAmount m_total_unspendables_genesis_block{0}; - CAmount m_total_unspendables_bip30{0}; - CAmount m_total_unspendables_scripts{0}; - CAmount m_total_unspendables_unclaimed_rewards{0}; + + CAmount m_block_subsidy{0}; + CAmount m_block_prevout_spent_amount{0}; + CAmount m_block_new_outputs_ex_coinbase_amount{0}; + CAmount m_block_coinbase_amount{0}; + + CAmount m_block_unspendables_genesis_block{0}; + CAmount m_block_unspendables_bip30{0}; + CAmount m_block_unspendables_scripts{0}; + CAmount m_block_unspendables_unclaimed_rewards{0}; [[nodiscard]] bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex); bool AllowPrune() const override { return true; } + uint32_t GetVersion() const override { return 1; } + bool MigrateToV1(); protected: bool CustomInit(const std::optional& block) override; diff --git a/src/index/txindex.h b/src/index/txindex.h index ef835fe5d7d..6d174af95a7 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -23,6 +23,7 @@ private: const std::unique_ptr m_db; bool AllowPrune() const override { return false; } + uint32_t GetVersion() const override { return 0; } protected: bool CustomAppend(const interfaces::BlockInfo& block) override; diff --git a/src/kernel/coinstats.h b/src/kernel/coinstats.h index c0c363a8428..b842ea3cc65 100644 --- a/src/kernel/coinstats.h +++ b/src/kernel/coinstats.h @@ -48,24 +48,26 @@ struct CCoinsStats { // Following values are only available from coinstats index - //! Total cumulative amount of block subsidies up to and including this block - CAmount total_subsidy{0}; - //! Total cumulative amount of unspendable coins up to and including this block + //! Amount of unspendable coins in this block CAmount total_unspendable_amount{0}; - //! Total cumulative amount of prevouts spent up to and including this block - CAmount total_prevout_spent_amount{0}; - //! Total cumulative amount of outputs created up to and including this block - CAmount total_new_outputs_ex_coinbase_amount{0}; - //! Total cumulative amount of coinbase outputs up to and including this block - CAmount total_coinbase_amount{0}; - //! The unspendable coinbase amount from the genesis block - CAmount total_unspendables_genesis_block{0}; - //! The two unspendable coinbase outputs total amount caused by BIP30 - CAmount total_unspendables_bip30{0}; - //! Total cumulative amount of outputs sent to unspendable scripts (OP_RETURN for example) up to and including this block - CAmount total_unspendables_scripts{0}; - //! Total cumulative amount of coins lost due to unclaimed miner rewards up to and including this block - CAmount total_unspendables_unclaimed_rewards{0}; + + //! Amount of block subsidies in this block + CAmount block_subsidy{0}; + //! Amount of prevouts spent in this block + CAmount block_prevout_spent_amount{0}; + //! Amount of outputs created in this block + CAmount block_new_outputs_ex_coinbase_amount{0}; + //! Amount of coinbase outputs in this block + CAmount block_coinbase_amount{0}; + + //! The unspendable coinbase output amount from the genesis block + CAmount block_unspendables_genesis_block{0}; + //! The unspendable coinbase output amounts caused by BIP30 + CAmount block_unspendables_bip30{0}; + //! Amount of outputs sent to unspendable scripts (OP_RETURN for example) in this block + CAmount block_unspendables_scripts{0}; + //! Amount of coins lost due to unclaimed miner rewards in this block + CAmount block_unspendables_unclaimed_rewards{0}; CCoinsStats() = default; CCoinsStats(int block_height, const uint256& block_hash); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 6e5c656f3d8..c37d081de1b 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1075,26 +1075,20 @@ static RPCHelpMan gettxoutsetinfo() } else { ret.pushKV("total_unspendable_amount", ValueFromAmount(stats.total_unspendable_amount)); - CCoinsStats prev_stats{}; - if (pindex->nHeight > 0) { - const std::optional maybe_prev_stats = GetUTXOStats(coins_view, *blockman, hash_type, node.rpc_interruption_point, pindex->pprev, index_requested); - if (!maybe_prev_stats) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); - } - prev_stats = maybe_prev_stats.value(); - } - UniValue block_info(UniValue::VOBJ); - block_info.pushKV("prevout_spent", ValueFromAmount(stats.total_prevout_spent_amount - prev_stats.total_prevout_spent_amount)); - block_info.pushKV("coinbase", ValueFromAmount(stats.total_coinbase_amount - prev_stats.total_coinbase_amount)); - block_info.pushKV("new_outputs_ex_coinbase", ValueFromAmount(stats.total_new_outputs_ex_coinbase_amount - prev_stats.total_new_outputs_ex_coinbase_amount)); - block_info.pushKV("unspendable", ValueFromAmount(stats.total_unspendable_amount - prev_stats.total_unspendable_amount)); + block_info.pushKV("prevout_spent", ValueFromAmount(stats.block_prevout_spent_amount)); + block_info.pushKV("coinbase", ValueFromAmount(stats.block_coinbase_amount)); + block_info.pushKV("new_outputs_ex_coinbase", ValueFromAmount(stats.block_new_outputs_ex_coinbase_amount)); + block_info.pushKV("unspendable", ValueFromAmount(stats.block_unspendables_genesis_block + + stats.block_unspendables_bip30 + + stats.block_unspendables_scripts + + stats.block_unspendables_unclaimed_rewards)); UniValue unspendables(UniValue::VOBJ); - unspendables.pushKV("genesis_block", ValueFromAmount(stats.total_unspendables_genesis_block - prev_stats.total_unspendables_genesis_block)); - unspendables.pushKV("bip30", ValueFromAmount(stats.total_unspendables_bip30 - prev_stats.total_unspendables_bip30)); - unspendables.pushKV("scripts", ValueFromAmount(stats.total_unspendables_scripts - prev_stats.total_unspendables_scripts)); - unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.total_unspendables_unclaimed_rewards - prev_stats.total_unspendables_unclaimed_rewards)); + unspendables.pushKV("genesis_block", ValueFromAmount(stats.block_unspendables_genesis_block)); + unspendables.pushKV("bip30", ValueFromAmount(stats.block_unspendables_bip30)); + unspendables.pushKV("scripts", ValueFromAmount(stats.block_unspendables_scripts)); + unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.block_unspendables_unclaimed_rewards)); block_info.pushKV("unspendables", std::move(unspendables)); ret.pushKV("block_info", std::move(block_info)); diff --git a/test/functional/data/coinstatsindex_v0/000003.log b/test/functional/data/coinstatsindex_v0/000003.log new file mode 100644 index 00000000000..224f50b1c94 Binary files /dev/null and b/test/functional/data/coinstatsindex_v0/000003.log differ diff --git a/test/functional/data/coinstatsindex_v0/CURRENT b/test/functional/data/coinstatsindex_v0/CURRENT new file mode 100644 index 00000000000..1a84852211e --- /dev/null +++ b/test/functional/data/coinstatsindex_v0/CURRENT @@ -0,0 +1 @@ +MANIFEST-000002 diff --git a/test/functional/data/coinstatsindex_v0/LOCK b/test/functional/data/coinstatsindex_v0/LOCK new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/functional/data/coinstatsindex_v0/MANIFEST-000002 b/test/functional/data/coinstatsindex_v0/MANIFEST-000002 new file mode 100644 index 00000000000..bbbc585686b Binary files /dev/null and b/test/functional/data/coinstatsindex_v0/MANIFEST-000002 differ diff --git a/test/functional/data/coinstatsindex_v0_corrupt/000003.log b/test/functional/data/coinstatsindex_v0_corrupt/000003.log new file mode 100644 index 00000000000..819c5b0a611 Binary files /dev/null and b/test/functional/data/coinstatsindex_v0_corrupt/000003.log differ diff --git a/test/functional/data/coinstatsindex_v0_corrupt/CURRENT b/test/functional/data/coinstatsindex_v0_corrupt/CURRENT new file mode 100644 index 00000000000..1a84852211e --- /dev/null +++ b/test/functional/data/coinstatsindex_v0_corrupt/CURRENT @@ -0,0 +1 @@ +MANIFEST-000002 diff --git a/test/functional/data/coinstatsindex_v0_corrupt/LOCK b/test/functional/data/coinstatsindex_v0_corrupt/LOCK new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/functional/data/coinstatsindex_v0_corrupt/MANIFEST-000002 b/test/functional/data/coinstatsindex_v0_corrupt/MANIFEST-000002 new file mode 100644 index 00000000000..bbbc585686b Binary files /dev/null and b/test/functional/data/coinstatsindex_v0_corrupt/MANIFEST-000002 differ diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py index a2363c4acfd..c2ce916bfc2 100755 --- a/test/functional/feature_coinstatsindex.py +++ b/test/functional/feature_coinstatsindex.py @@ -10,6 +10,9 @@ the index. """ from decimal import Decimal +import os +from pathlib import Path +import shutil from test_framework.blocktools import ( COINBASE_MATURITY, @@ -54,6 +57,11 @@ class CoinStatsIndexTest(BitcoinTestFramework): self._test_reorg_index() self._test_index_rejects_hash_serialized() self._test_init_index_after_reorg() + # Windows seems to require a different file format than we use in + # these tests + if os.name == 'posix': + self._test_outdated_index_version_migration() + self._test_outdated_index_version_corrupted() def block_sanity_check(self, block_info): block_subsidy = 50 @@ -71,8 +79,14 @@ class CoinStatsIndexTest(BitcoinTestFramework): # Both none and muhash options allow the usage of the index index_hash_options = ['none', 'muhash'] - # Generate a normal transaction and mine it + # Generate enough blocks so we have funds to spend. We make it + # deterministic so we can later use a different db to test + # migrating old versions of the index. + node.setmocktime(node.getblockheader(node.getbestblockhash())['time']) self.generate(self.wallet, COINBASE_MATURITY + 1) + node.setmocktime(0) + + # Generate a normal transaction and mine it self.wallet.send_self_transfer(from_node=node) self.generate(node, 1) @@ -323,6 +337,32 @@ class CoinStatsIndexTest(BitcoinTestFramework): res1 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=True) assert_equal(res["muhash"], res1["muhash"]) + def _test_outdated_index_version_migration(self): + self.log.info("Test a node with an outdated index version tries migration") + index_node = self.nodes[1] + self.stop_node(1) + index_db = index_node.chain_path / 'indexes' / 'coinstats' / 'db' + shutil.rmtree(index_db) + v0_index_db = Path(os.path.dirname(os.path.realpath(__file__))) / 'data' / 'coinstatsindex_v0' + shutil.copytree(v0_index_db, index_db) + msg1 = "Migrating coinstatsindex to new format (v1)." + msg2 = "Migration succeeded" + with index_node.assert_debug_log(expected_msgs=[msg1, msg2]): + self.restart_node(1, extra_args=["-coinstatsindex"]) + + def _test_outdated_index_version_corrupted(self): + self.log.info("Test a node doesn't start with an outdated index version that can not be migrated") + index_node = self.nodes[1] + self.stop_node(1) + index_db = index_node.chain_path / 'indexes' / 'coinstats' / 'db' + shutil.rmtree(index_db) + v0_index_db = Path(os.path.dirname(os.path.realpath(__file__))) / 'data' / 'coinstatsindex_v0_corrupt' + shutil.copytree(v0_index_db, index_db) + msg1 = "Coinstatsindex is corrupted" + msg2 = "Error while migrating to v1. In order to rebuild the index, remove the indexes/coinstats directory in your datadir" + with index_node.assert_debug_log(expected_msgs=[msg1, msg2]): + index_node.assert_start_raises_init_error(extra_args=["-coinstatsindex"]) + if __name__ == '__main__': CoinStatsIndexTest(__file__).main()