From 9b0ec1a7f9ffae816fd5ca32ff7e7559640b6f6d Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Tue, 15 May 2018 17:52:09 -0700 Subject: [PATCH 1/7] db: Remove obsolete methods from CBlockTreeDB. --- src/txdb.cpp | 11 ----------- src/txdb.h | 2 -- 2 files changed, 13 deletions(-) diff --git a/src/txdb.cpp b/src/txdb.cpp index 333d3596c1..90d937d4c0 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -237,17 +237,6 @@ bool CBlockTreeDB::WriteBatchSync(const std::vector >&vect) { - CDBBatch batch(*this); - for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) - batch.Write(std::make_pair(DB_TXINDEX, it->first), it->second); - return WriteBatch(batch); -} - bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } diff --git a/src/txdb.h b/src/txdb.h index 4193f98de1..3c509373f4 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -118,8 +118,6 @@ public: bool ReadLastBlockFile(int &nFile); bool WriteReindexing(bool fReindexing); bool ReadReindexing(bool &fReindexing); - bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); - bool WriteTxIndex(const std::vector > &vect); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function insertBlockIndex); From e5af5fc6fb4658599b940d1d50853129b31b8766 Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Tue, 15 May 2018 14:57:22 -0700 Subject: [PATCH 2/7] db: Make reusable base class for index databases. --- src/txdb.cpp | 10 +++++++--- src/txdb.h | 21 ++++++++++++++------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/txdb.cpp b/src/txdb.cpp index 90d937d4c0..624b23962a 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -415,8 +415,12 @@ bool CCoinsViewDB::Upgrade() { return !ShutdownRequested(); } +BaseIndexDB::BaseIndexDB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) : + CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate) +{} + TxIndexDB::TxIndexDB(size_t n_cache_size, bool f_memory, bool f_wipe) : - CDBWrapper(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe) + BaseIndexDB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe) {} bool TxIndexDB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const @@ -433,7 +437,7 @@ bool TxIndexDB::WriteTxs(const std::vector>& v_po return WriteBatch(batch); } -bool TxIndexDB::ReadBestBlock(CBlockLocator& locator) const +bool BaseIndexDB::ReadBestBlock(CBlockLocator& locator) const { bool success = Read(DB_BEST_BLOCK, locator); if (!success) { @@ -442,7 +446,7 @@ bool TxIndexDB::ReadBestBlock(CBlockLocator& locator) const return success; } -bool TxIndexDB::WriteBestBlock(const CBlockLocator& locator) +bool BaseIndexDB::WriteBestBlock(const CBlockLocator& locator) { return Write(DB_BEST_BLOCK, locator); } diff --git a/src/txdb.h b/src/txdb.h index 3c509373f4..f9d9e4246c 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -123,6 +123,19 @@ public: bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function insertBlockIndex); }; +class BaseIndexDB : public CDBWrapper +{ +public: + BaseIndexDB(const fs::path& path, size_t n_cache_size, + bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false); + + /// Read block locator of the chain that the index is in sync with. + bool ReadBestBlock(CBlockLocator& locator) const; + + /// Write block locator of the chain that the index is in sync with. + bool WriteBestBlock(const CBlockLocator& locator); +}; + /** * Access to the txindex database (indexes/txindex/) * @@ -132,7 +145,7 @@ public: * and block index entries may not be flushed to disk until after this database * is updated. */ -class TxIndexDB : public CDBWrapper +class TxIndexDB : public BaseIndexDB { public: explicit TxIndexDB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); @@ -144,12 +157,6 @@ public: /// Write a batch of transaction positions to the DB. bool WriteTxs(const std::vector>& v_pos); - /// Read block locator of the chain that the txindex is in sync with. - bool ReadBestBlock(CBlockLocator& locator) const; - - /// Write block locator of the chain that the txindex is in sync with. - bool WriteBestBlock(const CBlockLocator& locator); - /// Migrate txindex data from the block tree DB, where it may be for older nodes that have not /// been upgraded yet to the new database. bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator); From 61a1226d87d80234b2be123c5cad07534c318cfb Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Tue, 15 May 2018 14:47:37 -0700 Subject: [PATCH 3/7] index: Extract logic from TxIndex into reusable base class. --- src/index/txindex.cpp | 40 +++++++++++-------- src/index/txindex.h | 93 ++++++++++++++++++++++++++----------------- 2 files changed, 80 insertions(+), 53 deletions(-) diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 3ff16b7664..90de0fde97 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -28,11 +28,9 @@ static void FatalError(const char* fmt, const Args&... args) StartShutdown(); } -TxIndex::TxIndex(std::unique_ptr db) : - m_db(std::move(db)), m_synced(false), m_best_block_index(nullptr) -{} +TxIndex::TxIndex(std::unique_ptr db) : m_db(std::move(db)) {} -TxIndex::~TxIndex() +BaseIndex::~BaseIndex() { Interrupt(); Stop(); @@ -49,11 +47,17 @@ bool TxIndex::Init() return false; } + return BaseIndex::Init(); +} + +bool BaseIndex::Init() +{ CBlockLocator locator; - if (!m_db->ReadBestBlock(locator)) { + if (!GetDB().ReadBestBlock(locator)) { locator.SetNull(); } + LOCK(cs_main); m_best_block_index = FindForkInGlobalIndex(chainActive, locator); m_synced = m_best_block_index.load() == chainActive.Tip(); return true; @@ -75,7 +79,7 @@ static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev) return chainActive.Next(chainActive.FindFork(pindex_prev)); } -void TxIndex::ThreadSync() +void BaseIndex::ThreadSync() { const CBlockIndex* pindex = m_best_block_index.load(); if (!m_synced) { @@ -145,17 +149,19 @@ bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) return m_db->WriteTxs(vPos); } -bool TxIndex::WriteBestBlock(const CBlockIndex* block_index) +BaseIndexDB& TxIndex::GetDB() const { return *m_db; } + +bool BaseIndex::WriteBestBlock(const CBlockIndex* block_index) { LOCK(cs_main); - if (!m_db->WriteBestBlock(chainActive.GetLocator(block_index))) { + if (!GetDB().WriteBestBlock(chainActive.GetLocator(block_index))) { return error("%s: Failed to write locator to disk", __func__); } return true; } -void TxIndex::BlockConnected(const std::shared_ptr& block, const CBlockIndex* pindex, - const std::vector& txn_conflicted) +void BaseIndex::BlockConnected(const std::shared_ptr& block, const CBlockIndex* pindex, + const std::vector& txn_conflicted) { if (!m_synced) { return; @@ -192,7 +198,7 @@ void TxIndex::BlockConnected(const std::shared_ptr& block, const C } } -void TxIndex::ChainStateFlushed(const CBlockLocator& locator) +void BaseIndex::ChainStateFlushed(const CBlockLocator& locator) { if (!m_synced) { return; @@ -225,12 +231,12 @@ void TxIndex::ChainStateFlushed(const CBlockLocator& locator) return; } - if (!m_db->WriteBestBlock(locator)) { + if (!GetDB().WriteBestBlock(locator)) { error("%s: Failed to write locator to disk", __func__); } } -bool TxIndex::BlockUntilSyncedToCurrentChain() +bool BaseIndex::BlockUntilSyncedToCurrentChain() { AssertLockNotHeld(cs_main); @@ -282,12 +288,12 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe return true; } -void TxIndex::Interrupt() +void BaseIndex::Interrupt() { m_interrupt(); } -void TxIndex::Start() +void BaseIndex::Start() { // Need to register this ValidationInterface before running Init(), so that // callbacks are not missed if Init sets m_synced to true. @@ -298,10 +304,10 @@ void TxIndex::Start() } m_thread_sync = std::thread(&TraceThread>, "txindex", - std::bind(&TxIndex::ThreadSync, this)); + std::bind(&BaseIndex::ThreadSync, this)); } -void TxIndex::Stop() +void BaseIndex::Stop() { UnregisterValidationInterface(this); diff --git a/src/index/txindex.h b/src/index/txindex.h index 4937bd64e9..f38d845998 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -15,39 +15,31 @@ class CBlockIndex; /** - * TxIndex is used to look up transactions included in the blockchain by hash. - * The index is written to a LevelDB database and records the filesystem - * location of each transaction by transaction hash. + * Base class for indices of blockchain data. This implements + * CValidationInterface and ensures blocks are indexed sequentially according + * to their position in the active chain. */ -class TxIndex final : public CValidationInterface +class BaseIndex : public CValidationInterface { private: - const std::unique_ptr m_db; - /// Whether the index is in sync with the main chain. The flag is flipped /// from false to true once, after which point this starts processing /// ValidationInterface notifications to stay in sync. - std::atomic m_synced; + std::atomic m_synced{false}; - /// The last block in the chain that the TxIndex is in sync with. - std::atomic m_best_block_index; + /// The last block in the chain that the index is in sync with. + std::atomic m_best_block_index{nullptr}; std::thread m_thread_sync; CThreadInterrupt m_interrupt; - /// Initialize internal state from the database and block index. - bool Init(); - - /// Sync the tx index with the block index starting from the current best - /// block. Intended to be run in its own thread, m_thread_sync, and can be - /// interrupted with m_interrupt. Once the txindex gets in sync, the - /// m_synced flag is set and the BlockConnected ValidationInterface callback - /// takes over and the sync thread exits. + /// Sync the index with the block index starting from the current best block. + /// Intended to be run in its own thread, m_thread_sync, and can be + /// interrupted with m_interrupt. Once the index gets in sync, the m_synced + /// flag is set and the BlockConnected ValidationInterface callback takes + /// over and the sync thread exits. void ThreadSync(); - /// Write update index entries for a newly connected block. - bool WriteBlock(const CBlock& block, const CBlockIndex* pindex); - /// Write the current chain block locator to the DB. bool WriteBestBlock(const CBlockIndex* block_index); @@ -57,27 +49,25 @@ protected: void ChainStateFlushed(const CBlockLocator& locator) override; + /// Initialize internal state from the database and block index. + virtual bool Init(); + + /// Write update index entries for a newly connected block. + virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; } + + virtual BaseIndexDB& GetDB() const = 0; + public: - /// Constructs the TxIndex, which becomes available to be queried. - explicit TxIndex(std::unique_ptr db); - /// Destructor interrupts sync thread if running and blocks until it exits. - ~TxIndex(); + virtual ~BaseIndex(); - /// Blocks the current thread until the transaction index is caught up to - /// the current state of the block chain. This only blocks if the index has gotten in sync once - /// and only needs to process blocks in the ValidationInterface queue. If the index is catching - /// up from far behind, this method does not block and immediately returns false. + /// Blocks the current thread until the index is caught up to the current + /// state of the block chain. This only blocks if the index has gotten in + /// sync once and only needs to process blocks in the ValidationInterface + /// queue. If the index is catching up from far behind, this method does + /// not block and immediately returns false. bool BlockUntilSyncedToCurrentChain(); - /// Look up a transaction by hash. - /// - /// @param[in] tx_hash The hash of the transaction to be returned. - /// @param[out] block_hash The hash of the block the transaction is found in. - /// @param[out] tx The transaction itself. - /// @return true if transaction is found, false otherwise - bool FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const; - void Interrupt(); /// Start initializes the sync state and registers the instance as a @@ -88,6 +78,37 @@ public: void Stop(); }; +/** + * TxIndex is used to look up transactions included in the blockchain by hash. + * The index is written to a LevelDB database and records the filesystem + * location of each transaction by transaction hash. + */ +class TxIndex final : public BaseIndex +{ +private: + const std::unique_ptr m_db; + +protected: + /// Override base class init to migrate from old database. + bool Init() override; + + bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + + BaseIndexDB& GetDB() const override; + +public: + /// Constructs the index, which becomes available to be queried. + explicit TxIndex(std::unique_ptr db); + + /// Look up a transaction by hash. + /// + /// @param[in] tx_hash The hash of the transaction to be returned. + /// @param[out] block_hash The hash of the block the transaction is found in. + /// @param[out] tx The transaction itself. + /// @return true if transaction is found, false otherwise + bool FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const; +}; + /// The global transaction index, used in GetTransaction. May be null. extern std::unique_ptr g_txindex; From f376a4924109af2496b5fd16a787299eb039f1c8 Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Tue, 15 May 2018 15:45:20 -0700 Subject: [PATCH 4/7] index: Generalize logged statements in BaseIndex. --- src/index/txindex.cpp | 21 +++++++++++---------- src/index/txindex.h | 5 +++++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 90de0fde97..4434ce3f21 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -107,7 +107,8 @@ void BaseIndex::ThreadSync() int64_t current_time = GetTime(); if (last_log_time + SYNC_LOG_INTERVAL < current_time) { - LogPrintf("Syncing txindex with block chain from height %d\n", pindex->nHeight); + LogPrintf("Syncing %s with block chain from height %d\n", + GetName(), pindex->nHeight); last_log_time = current_time; } @@ -123,7 +124,7 @@ void BaseIndex::ThreadSync() return; } if (!WriteBlock(block, pindex)) { - FatalError("%s: Failed to write block %s to tx index database", + FatalError("%s: Failed to write block %s to index database", __func__, pindex->GetBlockHash().ToString()); return; } @@ -131,9 +132,9 @@ void BaseIndex::ThreadSync() } if (pindex) { - LogPrintf("txindex is enabled at height %d\n", pindex->nHeight); + LogPrintf("%s is enabled at height %d\n", GetName(), pindex->nHeight); } else { - LogPrintf("txindex is enabled\n"); + LogPrintf("%s is enabled\n", GetName()); } } @@ -182,7 +183,7 @@ void BaseIndex::BlockConnected(const std::shared_ptr& block, const // new chain tip. In this unlikely event, log a warning and let the queue clear. if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) { LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */ - "known best chain (tip=%s); not updating txindex\n", + "known best chain (tip=%s); not updating index\n", __func__, pindex->GetBlockHash().ToString(), best_block_index->GetBlockHash().ToString()); return; @@ -192,7 +193,7 @@ void BaseIndex::BlockConnected(const std::shared_ptr& block, const if (WriteBlock(*block, pindex)) { m_best_block_index = pindex; } else { - FatalError("%s: Failed to write block %s to txindex", + FatalError("%s: Failed to write block %s to index", __func__, pindex->GetBlockHash().ToString()); return; } @@ -225,7 +226,7 @@ void BaseIndex::ChainStateFlushed(const CBlockLocator& locator) const CBlockIndex* best_block_index = m_best_block_index.load(); if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) { LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */ - "chain (tip=%s); not writing txindex locator\n", + "chain (tip=%s); not writing index locator\n", __func__, locator_tip_hash.ToString(), best_block_index->GetBlockHash().ToString()); return; @@ -255,7 +256,7 @@ bool BaseIndex::BlockUntilSyncedToCurrentChain() } } - LogPrintf("%s: txindex is catching up on block notifications\n", __func__); + LogPrintf("%s: %s is catching up on block notifications\n", __func__, GetName()); SyncWithValidationInterfaceQueue(); return true; } @@ -299,11 +300,11 @@ void BaseIndex::Start() // callbacks are not missed if Init sets m_synced to true. RegisterValidationInterface(this); if (!Init()) { - FatalError("%s: txindex failed to initialize", __func__); + FatalError("%s: %s failed to initialize", __func__, GetName()); return; } - m_thread_sync = std::thread(&TraceThread>, "txindex", + m_thread_sync = std::thread(&TraceThread>, GetName(), std::bind(&BaseIndex::ThreadSync, this)); } diff --git a/src/index/txindex.h b/src/index/txindex.h index f38d845998..29626332ab 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -57,6 +57,9 @@ protected: virtual BaseIndexDB& GetDB() const = 0; + /// Get the name of the index for display in logs. + virtual const char* GetName() const = 0; + public: /// Destructor interrupts sync thread if running and blocks until it exits. virtual ~BaseIndex(); @@ -96,6 +99,8 @@ protected: BaseIndexDB& GetDB() const override; + const char* GetName() const override { return "txindex"; } + public: /// Constructs the index, which becomes available to be queried. explicit TxIndex(std::unique_ptr db); From 2318affd27de436ddf9d866a4b82eed8ea2e738b Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Tue, 15 May 2018 15:57:48 -0700 Subject: [PATCH 5/7] MOVEONLY: Move BaseIndex to its own file. --- src/Makefile.am | 2 + src/index/base.cpp | 258 ++++++++++++++++++++++++++++++++++++++++++ src/index/base.h | 84 ++++++++++++++ src/index/txindex.cpp | 251 ---------------------------------------- src/index/txindex.h | 76 +------------ 5 files changed, 345 insertions(+), 326 deletions(-) create mode 100644 src/index/base.cpp create mode 100644 src/index/base.h diff --git a/src/Makefile.am b/src/Makefile.am index 96e56915a6..9462c407a1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -107,6 +107,7 @@ BITCOIN_CORE_H = \ fs.h \ httprpc.h \ httpserver.h \ + index/base.h \ index/txindex.h \ indirectmap.h \ init.h \ @@ -208,6 +209,7 @@ libbitcoin_server_a_SOURCES = \ consensus/tx_verify.cpp \ httprpc.cpp \ httpserver.cpp \ + index/base.cpp \ index/txindex.cpp \ init.cpp \ dbwrapper.cpp \ diff --git a/src/index/base.cpp b/src/index/base.cpp new file mode 100644 index 0000000000..f381681a64 --- /dev/null +++ b/src/index/base.cpp @@ -0,0 +1,258 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds +constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds + +template +static void FatalError(const char* fmt, const Args&... args) +{ + std::string strMessage = tfm::format(fmt, args...); + SetMiscWarning(strMessage); + LogPrintf("*** %s\n", strMessage); + uiInterface.ThreadSafeMessageBox( + "Error: A fatal internal error occurred, see debug.log for details", + "", CClientUIInterface::MSG_ERROR); + StartShutdown(); +} + +BaseIndex::~BaseIndex() +{ + Interrupt(); + Stop(); +} + +bool BaseIndex::Init() +{ + CBlockLocator locator; + if (!GetDB().ReadBestBlock(locator)) { + locator.SetNull(); + } + + LOCK(cs_main); + m_best_block_index = FindForkInGlobalIndex(chainActive, locator); + m_synced = m_best_block_index.load() == chainActive.Tip(); + return true; +} + +static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev) +{ + AssertLockHeld(cs_main); + + if (!pindex_prev) { + return chainActive.Genesis(); + } + + const CBlockIndex* pindex = chainActive.Next(pindex_prev); + if (pindex) { + return pindex; + } + + return chainActive.Next(chainActive.FindFork(pindex_prev)); +} + +void BaseIndex::ThreadSync() +{ + const CBlockIndex* pindex = m_best_block_index.load(); + if (!m_synced) { + auto& consensus_params = Params().GetConsensus(); + + int64_t last_log_time = 0; + int64_t last_locator_write_time = 0; + while (true) { + if (m_interrupt) { + WriteBestBlock(pindex); + return; + } + + { + LOCK(cs_main); + const CBlockIndex* pindex_next = NextSyncBlock(pindex); + if (!pindex_next) { + WriteBestBlock(pindex); + m_best_block_index = pindex; + m_synced = true; + break; + } + pindex = pindex_next; + } + + int64_t current_time = GetTime(); + if (last_log_time + SYNC_LOG_INTERVAL < current_time) { + LogPrintf("Syncing %s with block chain from height %d\n", + GetName(), pindex->nHeight); + last_log_time = current_time; + } + + if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { + WriteBestBlock(pindex); + last_locator_write_time = current_time; + } + + CBlock block; + if (!ReadBlockFromDisk(block, pindex, consensus_params)) { + FatalError("%s: Failed to read block %s from disk", + __func__, pindex->GetBlockHash().ToString()); + return; + } + if (!WriteBlock(block, pindex)) { + FatalError("%s: Failed to write block %s to index database", + __func__, pindex->GetBlockHash().ToString()); + return; + } + } + } + + if (pindex) { + LogPrintf("%s is enabled at height %d\n", GetName(), pindex->nHeight); + } else { + LogPrintf("%s is enabled\n", GetName()); + } +} + +bool BaseIndex::WriteBestBlock(const CBlockIndex* block_index) +{ + LOCK(cs_main); + if (!GetDB().WriteBestBlock(chainActive.GetLocator(block_index))) { + return error("%s: Failed to write locator to disk", __func__); + } + return true; +} + +void BaseIndex::BlockConnected(const std::shared_ptr& block, const CBlockIndex* pindex, + const std::vector& txn_conflicted) +{ + if (!m_synced) { + return; + } + + const CBlockIndex* best_block_index = m_best_block_index.load(); + if (!best_block_index) { + if (pindex->nHeight != 0) { + FatalError("%s: First block connected is not the genesis block (height=%d)", + __func__, pindex->nHeight); + return; + } + } else { + // Ensure block connects to an ancestor of the current best block. This should be the case + // most of the time, but may not be immediately after the sync thread catches up and sets + // m_synced. Consider the case where there is a reorg and the blocks on the stale branch are + // in the ValidationInterface queue backlog even after the sync thread has caught up to the + // new chain tip. In this unlikely event, log a warning and let the queue clear. + if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) { + LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */ + "known best chain (tip=%s); not updating index\n", + __func__, pindex->GetBlockHash().ToString(), + best_block_index->GetBlockHash().ToString()); + return; + } + } + + if (WriteBlock(*block, pindex)) { + m_best_block_index = pindex; + } else { + FatalError("%s: Failed to write block %s to index", + __func__, pindex->GetBlockHash().ToString()); + return; + } +} + +void BaseIndex::ChainStateFlushed(const CBlockLocator& locator) +{ + if (!m_synced) { + return; + } + + const uint256& locator_tip_hash = locator.vHave.front(); + const CBlockIndex* locator_tip_index; + { + LOCK(cs_main); + locator_tip_index = LookupBlockIndex(locator_tip_hash); + } + + if (!locator_tip_index) { + FatalError("%s: First block (hash=%s) in locator was not found", + __func__, locator_tip_hash.ToString()); + return; + } + + // This checks that ChainStateFlushed callbacks are received after BlockConnected. The check may fail + // immediately after the sync thread catches up and sets m_synced. Consider the case where + // there is a reorg and the blocks on the stale branch are in the ValidationInterface queue + // backlog even after the sync thread has caught up to the new chain tip. In this unlikely + // event, log a warning and let the queue clear. + const CBlockIndex* best_block_index = m_best_block_index.load(); + if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) { + LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */ + "chain (tip=%s); not writing index locator\n", + __func__, locator_tip_hash.ToString(), + best_block_index->GetBlockHash().ToString()); + return; + } + + if (!GetDB().WriteBestBlock(locator)) { + error("%s: Failed to write locator to disk", __func__); + } +} + +bool BaseIndex::BlockUntilSyncedToCurrentChain() +{ + AssertLockNotHeld(cs_main); + + if (!m_synced) { + return false; + } + + { + // Skip the queue-draining stuff if we know we're caught up with + // chainActive.Tip(). + LOCK(cs_main); + const CBlockIndex* chain_tip = chainActive.Tip(); + const CBlockIndex* best_block_index = m_best_block_index.load(); + if (best_block_index->GetAncestor(chain_tip->nHeight) == chain_tip) { + return true; + } + } + + LogPrintf("%s: %s is catching up on block notifications\n", __func__, GetName()); + SyncWithValidationInterfaceQueue(); + return true; +} + +void BaseIndex::Interrupt() +{ + m_interrupt(); +} + +void BaseIndex::Start() +{ + // Need to register this ValidationInterface before running Init(), so that + // callbacks are not missed if Init sets m_synced to true. + RegisterValidationInterface(this); + if (!Init()) { + FatalError("%s: %s failed to initialize", __func__, GetName()); + return; + } + + m_thread_sync = std::thread(&TraceThread>, GetName(), + std::bind(&BaseIndex::ThreadSync, this)); +} + +void BaseIndex::Stop() +{ + UnregisterValidationInterface(this); + + if (m_thread_sync.joinable()) { + m_thread_sync.join(); + } +} diff --git a/src/index/base.h b/src/index/base.h new file mode 100644 index 0000000000..68efaaaf26 --- /dev/null +++ b/src/index/base.h @@ -0,0 +1,84 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_BASE_H +#define BITCOIN_INDEX_BASE_H + +#include +#include +#include +#include +#include +#include + +class CBlockIndex; + +/** + * Base class for indices of blockchain data. This implements + * CValidationInterface and ensures blocks are indexed sequentially according + * to their position in the active chain. + */ +class BaseIndex : public CValidationInterface +{ +private: + /// Whether the index is in sync with the main chain. The flag is flipped + /// from false to true once, after which point this starts processing + /// ValidationInterface notifications to stay in sync. + std::atomic m_synced{false}; + + /// The last block in the chain that the index is in sync with. + std::atomic m_best_block_index{nullptr}; + + std::thread m_thread_sync; + CThreadInterrupt m_interrupt; + + /// Sync the index with the block index starting from the current best block. + /// Intended to be run in its own thread, m_thread_sync, and can be + /// interrupted with m_interrupt. Once the index gets in sync, the m_synced + /// flag is set and the BlockConnected ValidationInterface callback takes + /// over and the sync thread exits. + void ThreadSync(); + + /// Write the current chain block locator to the DB. + bool WriteBestBlock(const CBlockIndex* block_index); + +protected: + void BlockConnected(const std::shared_ptr& block, const CBlockIndex* pindex, + const std::vector& txn_conflicted) override; + + void ChainStateFlushed(const CBlockLocator& locator) override; + + /// Initialize internal state from the database and block index. + virtual bool Init(); + + /// Write update index entries for a newly connected block. + virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; } + + virtual BaseIndexDB& GetDB() const = 0; + + /// Get the name of the index for display in logs. + virtual const char* GetName() const = 0; + +public: + /// Destructor interrupts sync thread if running and blocks until it exits. + virtual ~BaseIndex(); + + /// Blocks the current thread until the index is caught up to the current + /// state of the block chain. This only blocks if the index has gotten in + /// sync once and only needs to process blocks in the ValidationInterface + /// queue. If the index is catching up from far behind, this method does + /// not block and immediately returns false. + bool BlockUntilSyncedToCurrentChain(); + + void Interrupt(); + + /// Start initializes the sync state and registers the instance as a + /// ValidationInterface so that it stays in sync with blockchain updates. + void Start(); + + /// Stops the instance from staying in sync with blockchain updates. + void Stop(); +}; + +#endif // BITCOIN_INDEX_BASE_H diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 4434ce3f21..7d3d2fed52 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -2,40 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include #include -#include -#include -#include #include #include -#include - -constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds -constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds std::unique_ptr g_txindex; -template -static void FatalError(const char* fmt, const Args&... args) -{ - std::string strMessage = tfm::format(fmt, args...); - SetMiscWarning(strMessage); - LogPrintf("*** %s\n", strMessage); - uiInterface.ThreadSafeMessageBox( - "Error: A fatal internal error occurred, see debug.log for details", - "", CClientUIInterface::MSG_ERROR); - StartShutdown(); -} - TxIndex::TxIndex(std::unique_ptr db) : m_db(std::move(db)) {} -BaseIndex::~BaseIndex() -{ - Interrupt(); - Stop(); -} - bool TxIndex::Init() { LOCK(cs_main); @@ -50,94 +24,6 @@ bool TxIndex::Init() return BaseIndex::Init(); } -bool BaseIndex::Init() -{ - CBlockLocator locator; - if (!GetDB().ReadBestBlock(locator)) { - locator.SetNull(); - } - - LOCK(cs_main); - m_best_block_index = FindForkInGlobalIndex(chainActive, locator); - m_synced = m_best_block_index.load() == chainActive.Tip(); - return true; -} - -static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev) -{ - AssertLockHeld(cs_main); - - if (!pindex_prev) { - return chainActive.Genesis(); - } - - const CBlockIndex* pindex = chainActive.Next(pindex_prev); - if (pindex) { - return pindex; - } - - return chainActive.Next(chainActive.FindFork(pindex_prev)); -} - -void BaseIndex::ThreadSync() -{ - const CBlockIndex* pindex = m_best_block_index.load(); - if (!m_synced) { - auto& consensus_params = Params().GetConsensus(); - - int64_t last_log_time = 0; - int64_t last_locator_write_time = 0; - while (true) { - if (m_interrupt) { - WriteBestBlock(pindex); - return; - } - - { - LOCK(cs_main); - const CBlockIndex* pindex_next = NextSyncBlock(pindex); - if (!pindex_next) { - WriteBestBlock(pindex); - m_best_block_index = pindex; - m_synced = true; - break; - } - pindex = pindex_next; - } - - int64_t current_time = GetTime(); - if (last_log_time + SYNC_LOG_INTERVAL < current_time) { - LogPrintf("Syncing %s with block chain from height %d\n", - GetName(), pindex->nHeight); - last_log_time = current_time; - } - - if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { - WriteBestBlock(pindex); - last_locator_write_time = current_time; - } - - CBlock block; - if (!ReadBlockFromDisk(block, pindex, consensus_params)) { - FatalError("%s: Failed to read block %s from disk", - __func__, pindex->GetBlockHash().ToString()); - return; - } - if (!WriteBlock(block, pindex)) { - FatalError("%s: Failed to write block %s to index database", - __func__, pindex->GetBlockHash().ToString()); - return; - } - } - } - - if (pindex) { - LogPrintf("%s is enabled at height %d\n", GetName(), pindex->nHeight); - } else { - LogPrintf("%s is enabled\n", GetName()); - } -} - bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) { CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); @@ -152,115 +38,6 @@ bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) BaseIndexDB& TxIndex::GetDB() const { return *m_db; } -bool BaseIndex::WriteBestBlock(const CBlockIndex* block_index) -{ - LOCK(cs_main); - if (!GetDB().WriteBestBlock(chainActive.GetLocator(block_index))) { - return error("%s: Failed to write locator to disk", __func__); - } - return true; -} - -void BaseIndex::BlockConnected(const std::shared_ptr& block, const CBlockIndex* pindex, - const std::vector& txn_conflicted) -{ - if (!m_synced) { - return; - } - - const CBlockIndex* best_block_index = m_best_block_index.load(); - if (!best_block_index) { - if (pindex->nHeight != 0) { - FatalError("%s: First block connected is not the genesis block (height=%d)", - __func__, pindex->nHeight); - return; - } - } else { - // Ensure block connects to an ancestor of the current best block. This should be the case - // most of the time, but may not be immediately after the sync thread catches up and sets - // m_synced. Consider the case where there is a reorg and the blocks on the stale branch are - // in the ValidationInterface queue backlog even after the sync thread has caught up to the - // new chain tip. In this unlikely event, log a warning and let the queue clear. - if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) { - LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */ - "known best chain (tip=%s); not updating index\n", - __func__, pindex->GetBlockHash().ToString(), - best_block_index->GetBlockHash().ToString()); - return; - } - } - - if (WriteBlock(*block, pindex)) { - m_best_block_index = pindex; - } else { - FatalError("%s: Failed to write block %s to index", - __func__, pindex->GetBlockHash().ToString()); - return; - } -} - -void BaseIndex::ChainStateFlushed(const CBlockLocator& locator) -{ - if (!m_synced) { - return; - } - - const uint256& locator_tip_hash = locator.vHave.front(); - const CBlockIndex* locator_tip_index; - { - LOCK(cs_main); - locator_tip_index = LookupBlockIndex(locator_tip_hash); - } - - if (!locator_tip_index) { - FatalError("%s: First block (hash=%s) in locator was not found", - __func__, locator_tip_hash.ToString()); - return; - } - - // This checks that ChainStateFlushed callbacks are received after BlockConnected. The check may fail - // immediately after the sync thread catches up and sets m_synced. Consider the case where - // there is a reorg and the blocks on the stale branch are in the ValidationInterface queue - // backlog even after the sync thread has caught up to the new chain tip. In this unlikely - // event, log a warning and let the queue clear. - const CBlockIndex* best_block_index = m_best_block_index.load(); - if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) { - LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */ - "chain (tip=%s); not writing index locator\n", - __func__, locator_tip_hash.ToString(), - best_block_index->GetBlockHash().ToString()); - return; - } - - if (!GetDB().WriteBestBlock(locator)) { - error("%s: Failed to write locator to disk", __func__); - } -} - -bool BaseIndex::BlockUntilSyncedToCurrentChain() -{ - AssertLockNotHeld(cs_main); - - if (!m_synced) { - return false; - } - - { - // Skip the queue-draining stuff if we know we're caught up with - // chainActive.Tip(). - LOCK(cs_main); - const CBlockIndex* chain_tip = chainActive.Tip(); - const CBlockIndex* best_block_index = m_best_block_index.load(); - if (best_block_index->GetAncestor(chain_tip->nHeight) == chain_tip) { - return true; - } - } - - LogPrintf("%s: %s is catching up on block notifications\n", __func__, GetName()); - SyncWithValidationInterfaceQueue(); - return true; -} - bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const { CDiskTxPos postx; @@ -288,31 +65,3 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe block_hash = header.GetHash(); return true; } - -void BaseIndex::Interrupt() -{ - m_interrupt(); -} - -void BaseIndex::Start() -{ - // Need to register this ValidationInterface before running Init(), so that - // callbacks are not missed if Init sets m_synced to true. - RegisterValidationInterface(this); - if (!Init()) { - FatalError("%s: %s failed to initialize", __func__, GetName()); - return; - } - - m_thread_sync = std::thread(&TraceThread>, GetName(), - std::bind(&BaseIndex::ThreadSync, this)); -} - -void BaseIndex::Stop() -{ - UnregisterValidationInterface(this); - - if (m_thread_sync.joinable()) { - m_thread_sync.join(); - } -} diff --git a/src/index/txindex.h b/src/index/txindex.h index 29626332ab..fb92ad98dc 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -5,81 +5,7 @@ #ifndef BITCOIN_INDEX_TXINDEX_H #define BITCOIN_INDEX_TXINDEX_H -#include -#include -#include -#include -#include -#include - -class CBlockIndex; - -/** - * Base class for indices of blockchain data. This implements - * CValidationInterface and ensures blocks are indexed sequentially according - * to their position in the active chain. - */ -class BaseIndex : public CValidationInterface -{ -private: - /// Whether the index is in sync with the main chain. The flag is flipped - /// from false to true once, after which point this starts processing - /// ValidationInterface notifications to stay in sync. - std::atomic m_synced{false}; - - /// The last block in the chain that the index is in sync with. - std::atomic m_best_block_index{nullptr}; - - std::thread m_thread_sync; - CThreadInterrupt m_interrupt; - - /// Sync the index with the block index starting from the current best block. - /// Intended to be run in its own thread, m_thread_sync, and can be - /// interrupted with m_interrupt. Once the index gets in sync, the m_synced - /// flag is set and the BlockConnected ValidationInterface callback takes - /// over and the sync thread exits. - void ThreadSync(); - - /// Write the current chain block locator to the DB. - bool WriteBestBlock(const CBlockIndex* block_index); - -protected: - void BlockConnected(const std::shared_ptr& block, const CBlockIndex* pindex, - const std::vector& txn_conflicted) override; - - void ChainStateFlushed(const CBlockLocator& locator) override; - - /// Initialize internal state from the database and block index. - virtual bool Init(); - - /// Write update index entries for a newly connected block. - virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; } - - virtual BaseIndexDB& GetDB() const = 0; - - /// Get the name of the index for display in logs. - virtual const char* GetName() const = 0; - -public: - /// Destructor interrupts sync thread if running and blocks until it exits. - virtual ~BaseIndex(); - - /// Blocks the current thread until the index is caught up to the current - /// state of the block chain. This only blocks if the index has gotten in - /// sync once and only needs to process blocks in the ValidationInterface - /// queue. If the index is catching up from far behind, this method does - /// not block and immediately returns false. - bool BlockUntilSyncedToCurrentChain(); - - void Interrupt(); - - /// Start initializes the sync state and registers the instance as a - /// ValidationInterface so that it stays in sync with blockchain updates. - void Start(); - - /// Stops the instance from staying in sync with blockchain updates. - void Stop(); -}; +#include /** * TxIndex is used to look up transactions included in the blockchain by hash. From 89eddcd365e9a2218648f5cc5b9f22b28023f50a Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Tue, 15 May 2018 17:26:49 -0700 Subject: [PATCH 6/7] index: Remove TxIndexDB from public interface of TxIndex. --- src/index/txindex.cpp | 4 +++- src/index/txindex.h | 2 +- src/init.cpp | 3 +-- src/test/txindex_tests.cpp | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 7d3d2fed52..328039977f 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -8,7 +8,9 @@ std::unique_ptr g_txindex; -TxIndex::TxIndex(std::unique_ptr db) : m_db(std::move(db)) {} +TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe) + : m_db(MakeUnique(n_cache_size, f_memory, f_wipe)) +{} bool TxIndex::Init() { diff --git a/src/index/txindex.h b/src/index/txindex.h index fb92ad98dc..2a0c70e9d1 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -29,7 +29,7 @@ protected: public: /// Constructs the index, which becomes available to be queried. - explicit TxIndex(std::unique_ptr db); + explicit TxIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); /// Look up a transaction by hash. /// diff --git a/src/init.cpp b/src/init.cpp index b4e2eec0d2..9246f6e71c 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1606,8 +1606,7 @@ bool AppInitMain() // ********************************************************* Step 8: start indexers if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - auto txindex_db = MakeUnique(nTxIndexCache, false, fReindex); - g_txindex = MakeUnique(std::move(txindex_db)); + g_txindex = MakeUnique(nTxIndexCache, false, fReindex); g_txindex->Start(); } diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 14158f2875..be7ee2428b 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -15,7 +15,7 @@ BOOST_AUTO_TEST_SUITE(txindex_tests) BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) { - TxIndex txindex(MakeUnique(1 << 20, true)); + TxIndex txindex(1 << 20, true); CTransactionRef tx_disk; uint256 block_hash; From ec3073a274bf7affe1b8c87a10f75d126f5ac027 Mon Sep 17 00:00:00 2001 From: Jim Posen Date: Tue, 15 May 2018 17:20:17 -0700 Subject: [PATCH 7/7] index: Move index DBs into index/ directory. --- src/index/base.cpp | 20 ++++ src/index/base.h | 18 +++- src/index/txindex.cpp | 219 +++++++++++++++++++++++++++++++++++++++++- src/index/txindex.h | 12 ++- src/txdb.cpp | 176 --------------------------------- src/txdb.h | 64 ------------ 6 files changed, 264 insertions(+), 245 deletions(-) diff --git a/src/index/base.cpp b/src/index/base.cpp index f381681a64..738166dc94 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -11,6 +11,8 @@ #include #include +constexpr char DB_BEST_BLOCK = 'B'; + constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds @@ -26,6 +28,24 @@ static void FatalError(const char* fmt, const Args&... args) StartShutdown(); } +BaseIndex::DB::DB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) : + CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate) +{} + +bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const +{ + bool success = Read(DB_BEST_BLOCK, locator); + if (!success) { + locator.SetNull(); + } + return success; +} + +bool BaseIndex::DB::WriteBestBlock(const CBlockLocator& locator) +{ + return Write(DB_BEST_BLOCK, locator); +} + BaseIndex::~BaseIndex() { Interrupt(); diff --git a/src/index/base.h b/src/index/base.h index 68efaaaf26..04ee6e6cc2 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -5,10 +5,10 @@ #ifndef BITCOIN_INDEX_BASE_H #define BITCOIN_INDEX_BASE_H +#include #include #include #include -#include #include #include @@ -21,6 +21,20 @@ class CBlockIndex; */ class BaseIndex : public CValidationInterface { +protected: + class DB : public CDBWrapper + { + public: + DB(const fs::path& path, size_t n_cache_size, + bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false); + + /// Read block locator of the chain that the txindex is in sync with. + bool ReadBestBlock(CBlockLocator& locator) const; + + /// Write block locator of the chain that the txindex is in sync with. + bool WriteBestBlock(const CBlockLocator& locator); + }; + private: /// Whether the index is in sync with the main chain. The flag is flipped /// from false to true once, after which point this starts processing @@ -55,7 +69,7 @@ protected: /// Write update index entries for a newly connected block. virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; } - virtual BaseIndexDB& GetDB() const = 0; + virtual DB& GetDB() const = 0; /// Get the name of the index for display in logs. virtual const char* GetName() const = 0; diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 328039977f..e106b9b420 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -3,15 +3,232 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include +#include #include #include +#include + +constexpr char DB_BEST_BLOCK = 'B'; +constexpr char DB_TXINDEX = 't'; +constexpr char DB_TXINDEX_BLOCK = 'T'; + std::unique_ptr g_txindex; +struct CDiskTxPos : public CDiskBlockPos +{ + unsigned int nTxOffset; // after header + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITEAS(CDiskBlockPos, *this); + READWRITE(VARINT(nTxOffset)); + } + + CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { + } + + CDiskTxPos() { + SetNull(); + } + + void SetNull() { + CDiskBlockPos::SetNull(); + nTxOffset = 0; + } +}; + +/** + * Access to the txindex database (indexes/txindex/) + * + * The database stores a block locator of the chain the database is synced to + * so that the TxIndex can efficiently determine the point it last stopped at. + * A locator is used instead of a simple hash of the chain tip because blocks + * and block index entries may not be flushed to disk until after this database + * is updated. + */ +class TxIndex::DB : public BaseIndex::DB +{ +public: + explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + + /// Read the disk location of the transaction data with the given hash. Returns false if the + /// transaction hash is not indexed. + bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const; + + /// Write a batch of transaction positions to the DB. + bool WriteTxs(const std::vector>& v_pos); + + /// Migrate txindex data from the block tree DB, where it may be for older nodes that have not + /// been upgraded yet to the new database. + bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator); +}; + +TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) : + BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe) +{} + +bool TxIndex::DB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const +{ + return Read(std::make_pair(DB_TXINDEX, txid), pos); +} + +bool TxIndex::DB::WriteTxs(const std::vector>& v_pos) +{ + CDBBatch batch(*this); + for (const auto& tuple : v_pos) { + batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second); + } + return WriteBatch(batch); +} + +/* + * Safely persist a transfer of data from the old txindex database to the new one, and compact the + * range of keys updated. This is used internally by MigrateData. + */ +static void WriteTxIndexMigrationBatches(CDBWrapper& newdb, CDBWrapper& olddb, + CDBBatch& batch_newdb, CDBBatch& batch_olddb, + const std::pair& begin_key, + const std::pair& end_key) +{ + // Sync new DB changes to disk before deleting from old DB. + newdb.WriteBatch(batch_newdb, /*fSync=*/ true); + olddb.WriteBatch(batch_olddb); + olddb.CompactRange(begin_key, end_key); + + batch_newdb.Clear(); + batch_olddb.Clear(); +} + +bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator) +{ + // The prior implementation of txindex was always in sync with block index + // and presence was indicated with a boolean DB flag. If the flag is set, + // this means the txindex from a previous version is valid and in sync with + // the chain tip. The first step of the migration is to unset the flag and + // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the + // index entries are copied over in batches to the new database. Finally, + // DB_TXINDEX_BLOCK is erased from the old database and the block hash is + // written to the new database. + // + // Unsetting the boolean flag ensures that if the node is downgraded to a + // previous version, it will not see a corrupted, partially migrated index + // -- it will see that the txindex is disabled. When the node is upgraded + // again, the migration will pick up where it left off and sync to the block + // with hash DB_TXINDEX_BLOCK. + bool f_legacy_flag = false; + block_tree_db.ReadFlag("txindex", f_legacy_flag); + if (f_legacy_flag) { + if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) { + return error("%s: cannot write block indicator", __func__); + } + if (!block_tree_db.WriteFlag("txindex", false)) { + return error("%s: cannot write block index db flag", __func__); + } + } + + CBlockLocator locator; + if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) { + return true; + } + + int64_t count = 0; + LogPrintf("Upgrading txindex database... [0%%]\n"); + uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true); + int report_done = 0; + const size_t batch_size = 1 << 24; // 16 MiB + + CDBBatch batch_newdb(*this); + CDBBatch batch_olddb(block_tree_db); + + std::pair key; + std::pair begin_key{DB_TXINDEX, uint256()}; + std::pair prev_key = begin_key; + + bool interrupted = false; + std::unique_ptr cursor(block_tree_db.NewIterator()); + for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) { + boost::this_thread::interruption_point(); + if (ShutdownRequested()) { + interrupted = true; + break; + } + + if (!cursor->GetKey(key)) { + return error("%s: cannot get key from valid cursor", __func__); + } + if (key.first != DB_TXINDEX) { + break; + } + + // Log progress every 10%. + if (++count % 256 == 0) { + // Since txids are uniformly random and traversed in increasing order, the high 16 bits + // of the hash can be used to estimate the current progress. + const uint256& txid = key.second; + uint32_t high_nibble = + (static_cast(*(txid.begin() + 0)) << 8) + + (static_cast(*(txid.begin() + 1)) << 0); + int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5); + + uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true); + if (report_done < percentage_done/10) { + LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done); + report_done = percentage_done/10; + } + } + + CDiskTxPos value; + if (!cursor->GetValue(value)) { + return error("%s: cannot parse txindex record", __func__); + } + batch_newdb.Write(key, value); + batch_olddb.Erase(key); + + if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) { + // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating + // because LevelDB iterators are guaranteed to provide a consistent view of the + // underlying data, like a lightweight snapshot. + WriteTxIndexMigrationBatches(*this, block_tree_db, + batch_newdb, batch_olddb, + prev_key, key); + prev_key = key; + } + } + + // If these final DB batches complete the migration, write the best block + // hash marker to the new database and delete from the old one. This signals + // that the former is fully caught up to that point in the blockchain and + // that all txindex entries have been removed from the latter. + if (!interrupted) { + batch_olddb.Erase(DB_TXINDEX_BLOCK); + batch_newdb.Write(DB_BEST_BLOCK, locator); + } + + WriteTxIndexMigrationBatches(*this, block_tree_db, + batch_newdb, batch_olddb, + begin_key, key); + + if (interrupted) { + LogPrintf("[CANCELLED].\n"); + return false; + } + + uiInterface.ShowProgress("", 100, false); + + LogPrintf("[DONE].\n"); + return true; +} + TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe) : m_db(MakeUnique(n_cache_size, f_memory, f_wipe)) {} +TxIndex::~TxIndex() {} + bool TxIndex::Init() { LOCK(cs_main); @@ -38,7 +255,7 @@ bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) return m_db->WriteTxs(vPos); } -BaseIndexDB& TxIndex::GetDB() const { return *m_db; } +BaseIndex::DB& TxIndex::GetDB() const { return *m_db; } bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const { diff --git a/src/index/txindex.h b/src/index/txindex.h index 2a0c70e9d1..8202c3c951 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -5,7 +5,9 @@ #ifndef BITCOIN_INDEX_TXINDEX_H #define BITCOIN_INDEX_TXINDEX_H +#include #include +#include /** * TxIndex is used to look up transactions included in the blockchain by hash. @@ -14,8 +16,11 @@ */ class TxIndex final : public BaseIndex { +protected: + class DB; + private: - const std::unique_ptr m_db; + const std::unique_ptr m_db; protected: /// Override base class init to migrate from old database. @@ -23,7 +28,7 @@ protected: bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; - BaseIndexDB& GetDB() const override; + BaseIndex::DB& GetDB() const override; const char* GetName() const override { return "txindex"; } @@ -31,6 +36,9 @@ public: /// Constructs the index, which becomes available to be queried. explicit TxIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + // Destructor is declared because this class contains a unique_ptr to an incomplete type. + virtual ~TxIndex() override; + /// Look up a transaction by hash. /// /// @param[in] tx_hash The hash of the transaction to be returned. diff --git a/src/txdb.cpp b/src/txdb.cpp index 624b23962a..b1d5879c83 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -21,8 +21,6 @@ static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; -static const char DB_TXINDEX = 't'; -static const char DB_TXINDEX_BLOCK = 'T'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -414,177 +412,3 @@ bool CCoinsViewDB::Upgrade() { LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE"); return !ShutdownRequested(); } - -BaseIndexDB::BaseIndexDB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) : - CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate) -{} - -TxIndexDB::TxIndexDB(size_t n_cache_size, bool f_memory, bool f_wipe) : - BaseIndexDB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe) -{} - -bool TxIndexDB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const -{ - return Read(std::make_pair(DB_TXINDEX, txid), pos); -} - -bool TxIndexDB::WriteTxs(const std::vector>& v_pos) -{ - CDBBatch batch(*this); - for (const auto& tuple : v_pos) { - batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second); - } - return WriteBatch(batch); -} - -bool BaseIndexDB::ReadBestBlock(CBlockLocator& locator) const -{ - bool success = Read(DB_BEST_BLOCK, locator); - if (!success) { - locator.SetNull(); - } - return success; -} - -bool BaseIndexDB::WriteBestBlock(const CBlockLocator& locator) -{ - return Write(DB_BEST_BLOCK, locator); -} - -/* - * Safely persist a transfer of data from the old txindex database to the new one, and compact the - * range of keys updated. This is used internally by MigrateData. - */ -static void WriteTxIndexMigrationBatches(TxIndexDB& newdb, CBlockTreeDB& olddb, - CDBBatch& batch_newdb, CDBBatch& batch_olddb, - const std::pair& begin_key, - const std::pair& end_key) -{ - // Sync new DB changes to disk before deleting from old DB. - newdb.WriteBatch(batch_newdb, /*fSync=*/ true); - olddb.WriteBatch(batch_olddb); - olddb.CompactRange(begin_key, end_key); - - batch_newdb.Clear(); - batch_olddb.Clear(); -} - -bool TxIndexDB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator) -{ - // The prior implementation of txindex was always in sync with block index - // and presence was indicated with a boolean DB flag. If the flag is set, - // this means the txindex from a previous version is valid and in sync with - // the chain tip. The first step of the migration is to unset the flag and - // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the - // index entries are copied over in batches to the new database. Finally, - // DB_TXINDEX_BLOCK is erased from the old database and the block hash is - // written to the new database. - // - // Unsetting the boolean flag ensures that if the node is downgraded to a - // previous version, it will not see a corrupted, partially migrated index - // -- it will see that the txindex is disabled. When the node is upgraded - // again, the migration will pick up where it left off and sync to the block - // with hash DB_TXINDEX_BLOCK. - bool f_legacy_flag = false; - block_tree_db.ReadFlag("txindex", f_legacy_flag); - if (f_legacy_flag) { - if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) { - return error("%s: cannot write block indicator", __func__); - } - if (!block_tree_db.WriteFlag("txindex", false)) { - return error("%s: cannot write block index db flag", __func__); - } - } - - CBlockLocator locator; - if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) { - return true; - } - - int64_t count = 0; - LogPrintf("Upgrading txindex database... [0%%]\n"); - uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true); - int report_done = 0; - const size_t batch_size = 1 << 24; // 16 MiB - - CDBBatch batch_newdb(*this); - CDBBatch batch_olddb(block_tree_db); - - std::pair key; - std::pair begin_key{DB_TXINDEX, uint256()}; - std::pair prev_key = begin_key; - - bool interrupted = false; - std::unique_ptr cursor(block_tree_db.NewIterator()); - for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) { - boost::this_thread::interruption_point(); - if (ShutdownRequested()) { - interrupted = true; - break; - } - - if (!cursor->GetKey(key)) { - return error("%s: cannot get key from valid cursor", __func__); - } - if (key.first != DB_TXINDEX) { - break; - } - - // Log progress every 10%. - if (++count % 256 == 0) { - // Since txids are uniformly random and traversed in increasing order, the high 16 bits - // of the hash can be used to estimate the current progress. - const uint256& txid = key.second; - uint32_t high_nibble = - (static_cast(*(txid.begin() + 0)) << 8) + - (static_cast(*(txid.begin() + 1)) << 0); - int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5); - - uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true); - if (report_done < percentage_done/10) { - LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done); - report_done = percentage_done/10; - } - } - - CDiskTxPos value; - if (!cursor->GetValue(value)) { - return error("%s: cannot parse txindex record", __func__); - } - batch_newdb.Write(key, value); - batch_olddb.Erase(key); - - if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) { - // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating - // because LevelDB iterators are guaranteed to provide a consistent view of the - // underlying data, like a lightweight snapshot. - WriteTxIndexMigrationBatches(*this, block_tree_db, - batch_newdb, batch_olddb, - prev_key, key); - prev_key = key; - } - } - - // If these final DB batches complete the migration, write the best block - // hash marker to the new database and delete from the old one. This signals - // that the former is fully caught up to that point in the blockchain and - // that all txindex entries have been removed from the latter. - if (!interrupted) { - batch_olddb.Erase(DB_TXINDEX_BLOCK); - batch_newdb.Write(DB_BEST_BLOCK, locator); - } - - WriteTxIndexMigrationBatches(*this, block_tree_db, - batch_newdb, batch_olddb, - begin_key, key); - - if (interrupted) { - LogPrintf("[CANCELLED].\n"); - return false; - } - - uiInterface.ShowProgress("", 100, false); - - LogPrintf("[DONE].\n"); - return true; -} diff --git a/src/txdb.h b/src/txdb.h index f9d9e4246c..100adb428d 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -40,31 +40,6 @@ static const int64_t nMaxTxIndexCache = 1024; //! Max memory allocated to coin DB specific cache (MiB) static const int64_t nMaxCoinsDBCache = 8; -struct CDiskTxPos : public CDiskBlockPos -{ - unsigned int nTxOffset; // after header - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CDiskBlockPos, *this); - READWRITE(VARINT(nTxOffset)); - } - - CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { - } - - CDiskTxPos() { - SetNull(); - } - - void SetNull() { - CDiskBlockPos::SetNull(); - nTxOffset = 0; - } -}; - /** CCoinsView backed by the coin database (chainstate/) */ class CCoinsViewDB final : public CCoinsView { @@ -123,43 +98,4 @@ public: bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function insertBlockIndex); }; -class BaseIndexDB : public CDBWrapper -{ -public: - BaseIndexDB(const fs::path& path, size_t n_cache_size, - bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false); - - /// Read block locator of the chain that the index is in sync with. - bool ReadBestBlock(CBlockLocator& locator) const; - - /// Write block locator of the chain that the index is in sync with. - bool WriteBestBlock(const CBlockLocator& locator); -}; - -/** - * Access to the txindex database (indexes/txindex/) - * - * The database stores a block locator of the chain the database is synced to - * so that the TxIndex can efficiently determine the point it last stopped at. - * A locator is used instead of a simple hash of the chain tip because blocks - * and block index entries may not be flushed to disk until after this database - * is updated. - */ -class TxIndexDB : public BaseIndexDB -{ -public: - explicit TxIndexDB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); - - /// Read the disk location of the transaction data with the given hash. Returns false if the - /// transaction hash is not indexed. - bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const; - - /// Write a batch of transaction positions to the DB. - bool WriteTxs(const std::vector>& v_pos); - - /// Migrate txindex data from the block tree DB, where it may be for older nodes that have not - /// been upgraded yet to the new database. - bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator); -}; - #endif // BITCOIN_TXDB_H