Merge a5813613e9 into c5e44a0435

.gitignore

@@ -21,3 +21,6 @@ target/
 /guix-build-*
 
 /ci/scratch/
+
+# Test data that includes .log files
+!test/functional/data/coinstatsindex_v0/*

src/index/base.h

@@ -74,9 +74,6 @@ private:
     /// with an empty datadir if, e.g., `-txindex=1` is specified.
     std::atomic<bool> m_synced{false};
 
-    /// The last block in the chain that the index is in sync with.
-    std::atomic<const CBlockIndex*> m_best_block_index{nullptr};
-
     std::thread m_thread_sync;
     CThreadInterrupt m_interrupt;
 
@@ -94,11 +91,14 @@ private:
     bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip);
 
     virtual bool AllowPrune() const = 0;
+    virtual uint32_t GetVersion() const = 0;
 
     template <typename... Args>
     void FatalErrorf(util::ConstevalFormatString<sizeof...(Args)> fmt, const Args&... args);
 
 protected:
+    /// The last block in the chain that the index is in sync with.
+    std::atomic<const CBlockIndex*> m_best_block_index{nullptr};
     std::unique_ptr<interfaces::Chain> m_chain;
     Chainstate* m_chainstate{nullptr};
     const std::string m_name;

src/index/blockfilterindex.h

@@ -46,6 +46,7 @@ private:
     uint256 m_last_header{};
 
     bool AllowPrune() const override { return true; }
+    uint32_t GetVersion() const override { return 0; }
 
     bool Write(const BlockFilter& filter, uint32_t block_height, const uint256& filter_header);
 

src/index/coinstatsindex.cpp

@@ -23,6 +23,7 @@ using kernel::RemoveCoinHash;
 static constexpr uint8_t DB_BLOCK_HASH{'s'};
 static constexpr uint8_t DB_BLOCK_HEIGHT{'t'};
 static constexpr uint8_t DB_MUHASH{'M'};
+static constexpr uint8_t DB_VERSION{'V'};
 
 namespace {
 
@@ -31,15 +32,15 @@ struct DBVal {
     uint64_t transaction_output_count;
     uint64_t bogo_size;
     CAmount total_amount;
-    CAmount total_subsidy;
+    CAmount block_subsidy;
     CAmount total_unspendable_amount;
-    CAmount total_prevout_spent_amount;
-    CAmount total_new_outputs_ex_coinbase_amount;
-    CAmount total_coinbase_amount;
-    CAmount total_unspendables_genesis_block;
-    CAmount total_unspendables_bip30;
-    CAmount total_unspendables_scripts;
-    CAmount total_unspendables_unclaimed_rewards;
+    CAmount block_prevout_spent_amount;
+    CAmount block_new_outputs_ex_coinbase_amount;
+    CAmount block_coinbase_amount;
+    CAmount block_unspendables_genesis_block;
+    CAmount block_unspendables_bip30;
+    CAmount block_unspendables_scripts;
+    CAmount block_unspendables_unclaimed_rewards;
 
     SERIALIZE_METHODS(DBVal, obj)
     {
@@ -47,15 +48,15 @@ struct DBVal {
         READWRITE(obj.transaction_output_count);
         READWRITE(obj.bogo_size);
         READWRITE(obj.total_amount);
-        READWRITE(obj.total_subsidy);
+        READWRITE(obj.block_subsidy);
         READWRITE(obj.total_unspendable_amount);
-        READWRITE(obj.total_prevout_spent_amount);
-        READWRITE(obj.total_new_outputs_ex_coinbase_amount);
-        READWRITE(obj.total_coinbase_amount);
-        READWRITE(obj.total_unspendables_genesis_block);
-        READWRITE(obj.total_unspendables_bip30);
-        READWRITE(obj.total_unspendables_scripts);
-        READWRITE(obj.total_unspendables_unclaimed_rewards);
+        READWRITE(obj.block_prevout_spent_amount);
+        READWRITE(obj.block_new_outputs_ex_coinbase_amount);
+        READWRITE(obj.block_coinbase_amount);
+        READWRITE(obj.block_unspendables_genesis_block);
+        READWRITE(obj.block_unspendables_bip30);
+        READWRITE(obj.block_unspendables_scripts);
+        READWRITE(obj.block_unspendables_unclaimed_rewards);
     }
 };
 
@@ -114,9 +115,18 @@ CoinStatsIndex::CoinStatsIndex(std::unique_ptr<interfaces::Chain> chain, size_t
 
 bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block)
 {
-    CBlockUndo block_undo;
-    const CAmount block_subsidy{GetBlockSubsidy(block.height, Params().GetConsensus())};
-    m_total_subsidy += block_subsidy;
+    CAmount block_unspendable{0};
+    CBlockUndo block_undo{};
+
+    m_block_prevout_spent_amount = 0;
+    m_block_new_outputs_ex_coinbase_amount = 0;
+    m_block_coinbase_amount = 0;
+    m_block_subsidy = GetBlockSubsidy(block.height, Params().GetConsensus());
+
+    m_block_unspendables_genesis_block = 0;
+    m_block_unspendables_bip30 = 0;
+    m_block_unspendables_scripts = 0;
+    m_block_unspendables_unclaimed_rewards = 0;
 
     // Ignore genesis block
     if (block.height > 0) {
@@ -151,8 +161,8 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block)
 
             // Skip duplicate txid coinbase transactions (BIP30).
             if (IsBIP30Unspendable(*pindex) && tx->IsCoinBase()) {
-                m_total_unspendable_amount += block_subsidy;
-                m_total_unspendables_bip30 += block_subsidy;
+                block_unspendable += m_block_subsidy;
+                m_block_unspendables_bip30 += m_block_subsidy;
                 continue;
             }
 
@@ -163,17 +173,17 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block)
 
                 // Skip unspendable coins
                 if (coin.out.scriptPubKey.IsUnspendable()) {
-                    m_total_unspendable_amount += coin.out.nValue;
-                    m_total_unspendables_scripts += coin.out.nValue;
+                    block_unspendable += coin.out.nValue;
+                    m_block_unspendables_scripts += coin.out.nValue;
                     continue;
                 }
 
                 ApplyCoinHash(m_muhash, outpoint, coin);
 
                 if (tx->IsCoinBase()) {
-                    m_total_coinbase_amount += coin.out.nValue;
+                    m_block_coinbase_amount += coin.out.nValue;
                 } else {
-                    m_total_new_outputs_ex_coinbase_amount += coin.out.nValue;
+                    m_block_new_outputs_ex_coinbase_amount += coin.out.nValue;
                 }
 
                 ++m_transaction_output_count;
@@ -191,7 +201,7 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block)
 
                     RemoveCoinHash(m_muhash, outpoint, coin);
 
-                    m_total_prevout_spent_amount += coin.out.nValue;
+                    m_block_prevout_spent_amount += coin.out.nValue;
 
                     --m_transaction_output_count;
                     m_total_amount -= coin.out.nValue;
@@ -201,32 +211,33 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block)
         }
     } else {
         // genesis block
-        m_total_unspendable_amount += block_subsidy;
-        m_total_unspendables_genesis_block += block_subsidy;
+        block_unspendable += m_block_subsidy;
+        m_block_unspendables_genesis_block += m_block_subsidy;
     }
 
     // If spent prevouts + block subsidy are still a higher amount than
     // new outputs + coinbase + current unspendable amount this means
     // the miner did not claim the full block reward. Unclaimed block
     // rewards are also unspendable.
-    const CAmount unclaimed_rewards{(m_total_prevout_spent_amount + m_total_subsidy) - (m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount)};
-    m_total_unspendable_amount += unclaimed_rewards;
-    m_total_unspendables_unclaimed_rewards += unclaimed_rewards;
+    m_block_unspendables_unclaimed_rewards = (m_block_prevout_spent_amount + m_block_subsidy) - (m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + block_unspendable);
+    m_total_unspendable_amount += (m_block_unspendables_unclaimed_rewards + block_unspendable);
 
     std::pair<uint256, DBVal> value;
     value.first = block.hash;
     value.second.transaction_output_count = m_transaction_output_count;
     value.second.bogo_size = m_bogo_size;
     value.second.total_amount = m_total_amount;
-    value.second.total_subsidy = m_total_subsidy;
     value.second.total_unspendable_amount = m_total_unspendable_amount;
-    value.second.total_prevout_spent_amount = m_total_prevout_spent_amount;
-    value.second.total_new_outputs_ex_coinbase_amount = m_total_new_outputs_ex_coinbase_amount;
-    value.second.total_coinbase_amount = m_total_coinbase_amount;
-    value.second.total_unspendables_genesis_block = m_total_unspendables_genesis_block;
-    value.second.total_unspendables_bip30 = m_total_unspendables_bip30;
-    value.second.total_unspendables_scripts = m_total_unspendables_scripts;
-    value.second.total_unspendables_unclaimed_rewards = m_total_unspendables_unclaimed_rewards;
+
+    value.second.block_subsidy = m_block_subsidy;
+    value.second.block_prevout_spent_amount = m_block_prevout_spent_amount;
+    value.second.block_new_outputs_ex_coinbase_amount = m_block_new_outputs_ex_coinbase_amount;
+    value.second.block_coinbase_amount = m_block_coinbase_amount;
+
+    value.second.block_unspendables_genesis_block = m_block_unspendables_genesis_block;
+    value.second.block_unspendables_bip30 = m_block_unspendables_bip30;
+    value.second.block_unspendables_scripts = m_block_unspendables_scripts;
+    value.second.block_unspendables_unclaimed_rewards = m_block_unspendables_unclaimed_rewards;
 
     uint256 out;
     m_muhash.Finalize(out);
@@ -337,21 +348,51 @@ std::optional<CCoinsStats> CoinStatsIndex::LookUpStats(const CBlockIndex& block_
     stats.nTransactionOutputs = entry.transaction_output_count;
     stats.nBogoSize = entry.bogo_size;
     stats.total_amount = entry.total_amount;
-    stats.total_subsidy = entry.total_subsidy;
     stats.total_unspendable_amount = entry.total_unspendable_amount;
-    stats.total_prevout_spent_amount = entry.total_prevout_spent_amount;
-    stats.total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount;
-    stats.total_coinbase_amount = entry.total_coinbase_amount;
-    stats.total_unspendables_genesis_block = entry.total_unspendables_genesis_block;
-    stats.total_unspendables_bip30 = entry.total_unspendables_bip30;
-    stats.total_unspendables_scripts = entry.total_unspendables_scripts;
-    stats.total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards;
+
+    stats.block_subsidy = entry.block_subsidy;
+    stats.block_prevout_spent_amount = entry.block_prevout_spent_amount;
+    stats.block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount;
+    stats.block_coinbase_amount = entry.block_coinbase_amount;
+
+    stats.block_unspendables_genesis_block = entry.block_unspendables_genesis_block;
+    stats.block_unspendables_bip30 = entry.block_unspendables_bip30;
+    stats.block_unspendables_scripts = entry.block_unspendables_scripts;
+    stats.block_unspendables_unclaimed_rewards = entry.block_unspendables_unclaimed_rewards;
 
     return stats;
 }
 
 bool CoinStatsIndex::CustomInit(const std::optional<interfaces::BlockRef>& block)
 {
+    uint32_t code_version{GetVersion()};
+    uint32_t db_version{0};
+    // We are starting the index for the first time and write version first so
+    // we don't run into the version check later.
+    if (!block.has_value() && !m_db->Exists(DB_VERSION)) {
+        m_db->Write(DB_VERSION, code_version);
+        db_version = code_version;
+    }
+
+    // If we can't read a version this means the index has never been updated
+    // and needs to be reset now. Otherwise request a reset if we have a
+    // version mismatch.
+    if (m_db->Exists(DB_VERSION)) {
+        m_db->Read(DB_VERSION, db_version);
+    }
+    if (db_version == 0 && code_version == 1) {
+        // Attempt to migrate coinstatsindex without the need to reindex
+        if (!MigrateToV1()) {
+            LogError("%s migration: Error while migrating to v1. In order to rebuild the index, remove the indexes/coinstats directory in your datadir\n",
+                    GetName());
+            return false;
+        };
+    } else if (db_version != code_version) {
+        LogError("%s version mismatch: expected %s but %s was found. In order to rebuild the index, remove the indexes/coinstats directory in your datadir\n",
+                     GetName(), code_version, db_version);
+        return false;
+    }
+
     if (!m_db->Read(DB_MUHASH, m_muhash)) {
         // Check that the cause of the read failure is that the key does not
         // exist. Any other errors indicate database corruption or a disk
@@ -382,15 +423,17 @@ bool CoinStatsIndex::CustomInit(const std::optional<interfaces::BlockRef>& block
         m_transaction_output_count = entry.transaction_output_count;
         m_bogo_size = entry.bogo_size;
         m_total_amount = entry.total_amount;
-        m_total_subsidy = entry.total_subsidy;
         m_total_unspendable_amount = entry.total_unspendable_amount;
-        m_total_prevout_spent_amount = entry.total_prevout_spent_amount;
-        m_total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount;
-        m_total_coinbase_amount = entry.total_coinbase_amount;
-        m_total_unspendables_genesis_block = entry.total_unspendables_genesis_block;
-        m_total_unspendables_bip30 = entry.total_unspendables_bip30;
-        m_total_unspendables_scripts = entry.total_unspendables_scripts;
-        m_total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards;
+
+        m_block_subsidy = entry.block_subsidy;
+        m_block_prevout_spent_amount = entry.block_prevout_spent_amount;
+        m_block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount;
+        m_block_coinbase_amount = entry.block_coinbase_amount;
+
+        m_block_unspendables_genesis_block = entry.block_unspendables_genesis_block;
+        m_block_unspendables_bip30 = entry.block_unspendables_bip30;
+        m_block_unspendables_scripts = entry.block_unspendables_scripts;
+        m_block_unspendables_unclaimed_rewards = entry.block_unspendables_unclaimed_rewards;
     }
 
     return true;
@@ -410,9 +453,6 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex
     CBlockUndo block_undo;
     std::pair<uint256, DBVal> read_out;
 
-    const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())};
-    m_total_subsidy -= block_subsidy;
-
     // Ignore genesis block
     if (pindex->nHeight > 0) {
         if (!m_chainstate->m_blockman.ReadBlockUndo(block_undo, *pindex)) {
@@ -448,18 +488,11 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex
             // Skip unspendable coins
             if (coin.out.scriptPubKey.IsUnspendable()) {
                 m_total_unspendable_amount -= coin.out.nValue;
-                m_total_unspendables_scripts -= coin.out.nValue;
                 continue;
             }
 
             RemoveCoinHash(m_muhash, outpoint, coin);
 
-            if (tx->IsCoinBase()) {
-                m_total_coinbase_amount -= coin.out.nValue;
-            } else {
-                m_total_new_outputs_ex_coinbase_amount -= coin.out.nValue;
-            }
-
             --m_transaction_output_count;
             m_total_amount -= coin.out.nValue;
             m_bogo_size -= GetBogoSize(coin.out.scriptPubKey);
@@ -475,8 +508,6 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex
 
                 ApplyCoinHash(m_muhash, outpoint, coin);
 
-                m_total_prevout_spent_amount -= coin.out.nValue;
-
                 m_transaction_output_count++;
                 m_total_amount += coin.out.nValue;
                 m_bogo_size += GetBogoSize(coin.out.scriptPubKey);
@@ -484,27 +515,81 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex
         }
     }
 
-    const CAmount unclaimed_rewards{(m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount) - (m_total_prevout_spent_amount + m_total_subsidy)};
-    m_total_unspendable_amount -= unclaimed_rewards;
-    m_total_unspendables_unclaimed_rewards -= unclaimed_rewards;
-
-    // Check that the rolled back internal values are consistent with the DB read out
+    // Check that the rolled back internal values are consistent with the DB
+    // read out where possible, i.e. when total historical values are tracked.
+    // Otherwise just read the values from the index entry.
     uint256 out;
     m_muhash.Finalize(out);
     Assert(read_out.second.muhash == out);
 
-    Assert(m_transaction_output_count == read_out.second.transaction_output_count);
     Assert(m_total_amount == read_out.second.total_amount);
-    Assert(m_bogo_size == read_out.second.bogo_size);
-    Assert(m_total_subsidy == read_out.second.total_subsidy);
+    m_total_unspendable_amount -= m_block_unspendables_unclaimed_rewards;
     Assert(m_total_unspendable_amount == read_out.second.total_unspendable_amount);
-    Assert(m_total_prevout_spent_amount == read_out.second.total_prevout_spent_amount);
-    Assert(m_total_new_outputs_ex_coinbase_amount == read_out.second.total_new_outputs_ex_coinbase_amount);
-    Assert(m_total_coinbase_amount == read_out.second.total_coinbase_amount);
-    Assert(m_total_unspendables_genesis_block == read_out.second.total_unspendables_genesis_block);
-    Assert(m_total_unspendables_bip30 == read_out.second.total_unspendables_bip30);
-    Assert(m_total_unspendables_scripts == read_out.second.total_unspendables_scripts);
-    Assert(m_total_unspendables_unclaimed_rewards == read_out.second.total_unspendables_unclaimed_rewards);
+    Assert(m_transaction_output_count == read_out.second.transaction_output_count);
+    Assert(m_bogo_size == read_out.second.bogo_size);
+
+    m_block_subsidy = read_out.second.block_subsidy;
+    m_block_prevout_spent_amount = read_out.second.block_prevout_spent_amount;
+    m_block_new_outputs_ex_coinbase_amount = read_out.second.block_new_outputs_ex_coinbase_amount;
+    m_block_coinbase_amount = read_out.second.block_coinbase_amount;
+
+    m_block_unspendables_genesis_block = read_out.second.block_unspendables_genesis_block;
+    m_block_unspendables_bip30 = read_out.second.block_unspendables_bip30;
+    m_block_unspendables_scripts = read_out.second.block_unspendables_scripts;
+    m_block_unspendables_unclaimed_rewards = read_out.second.block_unspendables_unclaimed_rewards;
 
     return true;
 }
+
+bool CoinStatsIndex::MigrateToV1() {
+    LogInfo("Migrating coinstatsindex to new format (v1). This might take a few minutes.\n");
+    CDBBatch batch(*m_db);
+    DBVal entry;
+    DBVal entry_prev;
+    const CBlockIndex *pindex{m_best_block_index};
+    if (!LookUpOne(*m_db, {pindex->GetBlockHash(), pindex->nHeight}, entry)) {
+        return false;
+    }
+
+    // Loop backwards until we hit the genesis block which doesn't need to be updated
+    while (pindex->pprev) {
+        if (pindex->nHeight % 10000 == 0) LogInfo("Migrating block at height %i\n", pindex->nHeight);
+        // Load Previous entry
+        if (!LookUpOne(*m_db, {pindex->pprev->GetBlockHash(), pindex->pprev->nHeight}, entry_prev)) {
+            LogError("Coinstatsindex is corrupted at height %i\n", pindex->pprev->nHeight);
+            return false;
+        }
+        // Combine entries
+        if (entry.block_subsidy < entry_prev.block_subsidy
+            || entry.block_prevout_spent_amount < entry_prev.block_prevout_spent_amount
+            || entry.block_prevout_spent_amount < entry_prev.block_prevout_spent_amount
+            || entry.block_new_outputs_ex_coinbase_amount < entry_prev.block_new_outputs_ex_coinbase_amount
+            || entry.block_coinbase_amount < entry_prev.block_coinbase_amount
+            || entry.block_unspendables_genesis_block < entry_prev.block_unspendables_genesis_block
+            || entry.block_unspendables_bip30 < entry_prev.block_unspendables_bip30
+            || entry.block_unspendables_scripts < entry_prev.block_unspendables_scripts
+            || entry.block_unspendables_unclaimed_rewards < entry_prev.block_unspendables_unclaimed_rewards
+        ) {
+            LogError("Coinstatsindex is corrupted at height %i\n", pindex->nHeight);
+            return false;
+        }
+        entry.block_subsidy = entry.block_subsidy - entry_prev.block_subsidy;
+        entry.block_prevout_spent_amount = entry.block_prevout_spent_amount - entry_prev.block_prevout_spent_amount;
+        entry.block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount - entry_prev.block_new_outputs_ex_coinbase_amount;
+        entry.block_coinbase_amount = entry.block_coinbase_amount - entry_prev.block_coinbase_amount;
+        entry.block_unspendables_genesis_block = entry.block_unspendables_genesis_block - entry_prev.block_unspendables_genesis_block;
+        entry.block_unspendables_bip30 = entry.block_unspendables_bip30 - entry_prev.block_unspendables_bip30;
+        entry.block_unspendables_scripts = entry.block_unspendables_scripts - entry_prev.block_unspendables_scripts;
+        entry.block_unspendables_unclaimed_rewards = entry.block_unspendables_unclaimed_rewards - entry_prev.block_unspendables_unclaimed_rewards;
+        std::pair<uint256, DBVal> result;
+        result.first = pindex->GetBlockHash();
+        result.second = entry;
+        batch.Write(DBHeightKey(pindex->nHeight), result);
+        pindex = pindex->pprev;
+        entry = entry_prev;
+    }
+    batch.Write(DB_VERSION, 1);
+    if (!m_db->WriteBatch(batch)) return false;
+    LogInfo("Migration succeeded\n");
+    return true;
+}

src/index/coinstatsindex.h

@@ -28,19 +28,23 @@ private:
     uint64_t m_transaction_output_count{0};
     uint64_t m_bogo_size{0};
     CAmount m_total_amount{0};
-    CAmount m_total_subsidy{0};
     CAmount m_total_unspendable_amount{0};
-    CAmount m_total_prevout_spent_amount{0};
-    CAmount m_total_new_outputs_ex_coinbase_amount{0};
-    CAmount m_total_coinbase_amount{0};
-    CAmount m_total_unspendables_genesis_block{0};
-    CAmount m_total_unspendables_bip30{0};
-    CAmount m_total_unspendables_scripts{0};
-    CAmount m_total_unspendables_unclaimed_rewards{0};
+
+    CAmount m_block_subsidy{0};
+    CAmount m_block_prevout_spent_amount{0};
+    CAmount m_block_new_outputs_ex_coinbase_amount{0};
+    CAmount m_block_coinbase_amount{0};
+
+    CAmount m_block_unspendables_genesis_block{0};
+    CAmount m_block_unspendables_bip30{0};
+    CAmount m_block_unspendables_scripts{0};
+    CAmount m_block_unspendables_unclaimed_rewards{0};
 
     [[nodiscard]] bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex);
 
     bool AllowPrune() const override { return true; }
+    uint32_t GetVersion() const override { return 1; }
+    bool MigrateToV1();
 
 protected:
     bool CustomInit(const std::optional<interfaces::BlockRef>& block) override;

src/index/txindex.h

@@ -23,6 +23,7 @@ private:
     const std::unique_ptr<DB> m_db;
 
     bool AllowPrune() const override { return false; }
+    uint32_t GetVersion() const override { return 0; }
 
 protected:
     bool CustomAppend(const interfaces::BlockInfo& block) override;

src/kernel/coinstats.h

@@ -48,24 +48,26 @@ struct CCoinsStats {
 
     // Following values are only available from coinstats index
 
-    //! Total cumulative amount of block subsidies up to and including this block
-    CAmount total_subsidy{0};
-    //! Total cumulative amount of unspendable coins up to and including this block
+    //! Amount of unspendable coins in this block
     CAmount total_unspendable_amount{0};
-    //! Total cumulative amount of prevouts spent up to and including this block
-    CAmount total_prevout_spent_amount{0};
-    //! Total cumulative amount of outputs created up to and including this block
-    CAmount total_new_outputs_ex_coinbase_amount{0};
-    //! Total cumulative amount of coinbase outputs up to and including this block
-    CAmount total_coinbase_amount{0};
-    //! The unspendable coinbase amount from the genesis block
-    CAmount total_unspendables_genesis_block{0};
-    //! The two unspendable coinbase outputs total amount caused by BIP30
-    CAmount total_unspendables_bip30{0};
-    //! Total cumulative amount of outputs sent to unspendable scripts (OP_RETURN for example) up to and including this block
-    CAmount total_unspendables_scripts{0};
-    //! Total cumulative amount of coins lost due to unclaimed miner rewards up to and including this block
-    CAmount total_unspendables_unclaimed_rewards{0};
+
+    //! Amount of block subsidies in this block
+    CAmount block_subsidy{0};
+    //! Amount of prevouts spent in this block
+    CAmount block_prevout_spent_amount{0};
+    //! Amount of outputs created in this block
+    CAmount block_new_outputs_ex_coinbase_amount{0};
+    //! Amount of coinbase outputs in this block
+    CAmount block_coinbase_amount{0};
+
+    //! The unspendable coinbase output amount from the genesis block
+    CAmount block_unspendables_genesis_block{0};
+    //! The unspendable coinbase output amounts caused by BIP30
+    CAmount block_unspendables_bip30{0};
+    //! Amount of outputs sent to unspendable scripts (OP_RETURN for example) in this block
+    CAmount block_unspendables_scripts{0};
+    //! Amount of coins lost due to unclaimed miner rewards in this block
+    CAmount block_unspendables_unclaimed_rewards{0};
 
     CCoinsStats() = default;
     CCoinsStats(int block_height, const uint256& block_hash);

src/rpc/blockchain.cpp

@@ -1075,26 +1075,20 @@ static RPCHelpMan gettxoutsetinfo()
         } else {
             ret.pushKV("total_unspendable_amount", ValueFromAmount(stats.total_unspendable_amount));
 
-            CCoinsStats prev_stats{};
-            if (pindex->nHeight > 0) {
-                const std::optional<CCoinsStats> maybe_prev_stats = GetUTXOStats(coins_view, *blockman, hash_type, node.rpc_interruption_point, pindex->pprev, index_requested);
-                if (!maybe_prev_stats) {
-                    throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set");
-                }
-                prev_stats = maybe_prev_stats.value();
-            }
-
             UniValue block_info(UniValue::VOBJ);
-            block_info.pushKV("prevout_spent", ValueFromAmount(stats.total_prevout_spent_amount - prev_stats.total_prevout_spent_amount));
-            block_info.pushKV("coinbase", ValueFromAmount(stats.total_coinbase_amount - prev_stats.total_coinbase_amount));
-            block_info.pushKV("new_outputs_ex_coinbase", ValueFromAmount(stats.total_new_outputs_ex_coinbase_amount - prev_stats.total_new_outputs_ex_coinbase_amount));
-            block_info.pushKV("unspendable", ValueFromAmount(stats.total_unspendable_amount - prev_stats.total_unspendable_amount));
+            block_info.pushKV("prevout_spent", ValueFromAmount(stats.block_prevout_spent_amount));
+            block_info.pushKV("coinbase", ValueFromAmount(stats.block_coinbase_amount));
+            block_info.pushKV("new_outputs_ex_coinbase", ValueFromAmount(stats.block_new_outputs_ex_coinbase_amount));
+            block_info.pushKV("unspendable", ValueFromAmount(stats.block_unspendables_genesis_block +
+                        stats.block_unspendables_bip30 +
+                        stats.block_unspendables_scripts +
+                        stats.block_unspendables_unclaimed_rewards));
 
             UniValue unspendables(UniValue::VOBJ);
-            unspendables.pushKV("genesis_block", ValueFromAmount(stats.total_unspendables_genesis_block - prev_stats.total_unspendables_genesis_block));
-            unspendables.pushKV("bip30", ValueFromAmount(stats.total_unspendables_bip30 - prev_stats.total_unspendables_bip30));
-            unspendables.pushKV("scripts", ValueFromAmount(stats.total_unspendables_scripts - prev_stats.total_unspendables_scripts));
-            unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.total_unspendables_unclaimed_rewards - prev_stats.total_unspendables_unclaimed_rewards));
+            unspendables.pushKV("genesis_block", ValueFromAmount(stats.block_unspendables_genesis_block));
+            unspendables.pushKV("bip30", ValueFromAmount(stats.block_unspendables_bip30));
+            unspendables.pushKV("scripts", ValueFromAmount(stats.block_unspendables_scripts));
+            unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.block_unspendables_unclaimed_rewards));
             block_info.pushKV("unspendables", std::move(unspendables));
 
             ret.pushKV("block_info", std::move(block_info));

test/functional/data/coinstatsindex_v0/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000002

test/functional/data/coinstatsindex_v0_corrupt/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000002

test/functional/feature_coinstatsindex.py

@@ -10,6 +10,9 @@ the index.
 """
 
 from decimal import Decimal
+import os
+from pathlib import Path
+import shutil
 
 from test_framework.blocktools import (
     COINBASE_MATURITY,
@@ -54,6 +57,11 @@ class CoinStatsIndexTest(BitcoinTestFramework):
         self._test_reorg_index()
         self._test_index_rejects_hash_serialized()
         self._test_init_index_after_reorg()
+        # Windows seems to require a different file format than we use in
+        # these tests
+        if os.name == 'posix':
+            self._test_outdated_index_version_migration()
+            self._test_outdated_index_version_corrupted()
 
     def block_sanity_check(self, block_info):
         block_subsidy = 50
@@ -71,8 +79,14 @@ class CoinStatsIndexTest(BitcoinTestFramework):
         # Both none and muhash options allow the usage of the index
         index_hash_options = ['none', 'muhash']
 
-        # Generate a normal transaction and mine it
+        # Generate enough blocks so we have funds to spend. We make it
+        # deterministic so we can later use a different db to test
+        # migrating old versions of the index.
+        node.setmocktime(node.getblockheader(node.getbestblockhash())['time'])
         self.generate(self.wallet, COINBASE_MATURITY + 1)
+        node.setmocktime(0)
+
+        # Generate a normal transaction and mine it
         self.wallet.send_self_transfer(from_node=node)
         self.generate(node, 1)
 
@@ -323,6 +337,32 @@ class CoinStatsIndexTest(BitcoinTestFramework):
         res1 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=True)
         assert_equal(res["muhash"], res1["muhash"])
 
+    def _test_outdated_index_version_migration(self):
+        self.log.info("Test a node with an outdated index version tries migration")
+        index_node = self.nodes[1]
+        self.stop_node(1)
+        index_db = index_node.chain_path / 'indexes' / 'coinstats' / 'db'
+        shutil.rmtree(index_db)
+        v0_index_db = Path(os.path.dirname(os.path.realpath(__file__))) / 'data' / 'coinstatsindex_v0'
+        shutil.copytree(v0_index_db, index_db)
+        msg1 = "Migrating coinstatsindex to new format (v1)."
+        msg2 = "Migration succeeded"
+        with index_node.assert_debug_log(expected_msgs=[msg1, msg2]):
+            self.restart_node(1, extra_args=["-coinstatsindex"])
+
+    def _test_outdated_index_version_corrupted(self):
+        self.log.info("Test a node doesn't start with an outdated index version that can not be migrated")
+        index_node = self.nodes[1]
+        self.stop_node(1)
+        index_db = index_node.chain_path / 'indexes' / 'coinstats' / 'db'
+        shutil.rmtree(index_db)
+        v0_index_db = Path(os.path.dirname(os.path.realpath(__file__))) / 'data' / 'coinstatsindex_v0_corrupt'
+        shutil.copytree(v0_index_db, index_db)
+        msg1 = "Coinstatsindex is corrupted"
+        msg2 = "Error while migrating to v1. In order to rebuild the index, remove the indexes/coinstats directory in your datadir"
+        with index_node.assert_debug_log(expected_msgs=[msg1, msg2]):
+            index_node.assert_start_raises_init_error(extra_args=["-coinstatsindex"])
+
 
 if __name__ == '__main__':
     CoinStatsIndexTest(__file__).main()