Merge 578654c686 into 433412fd84

doc/release-notes-31483.md

@@ -0,0 +1,6 @@
+Updated settings
+------
+
+- On 32-bit systems an error is returned on startup if the user passes a
+  `-dbcache` value exceeding 4GiB.
+

src/bitcoin-chainstate.cpp

@@ -19,9 +19,9 @@
 
 #include <consensus/validation.h>
 #include <core_io.h>
+#include <kernel/caches.h>
 #include <logging.h>
 #include <node/blockstorage.h>
-#include <node/caches.h>
 #include <node/chainstate.h>
 #include <random.h>
 #include <script/sigcache.h>
@@ -123,10 +123,7 @@ int main(int argc, char* argv[])
     util::SignalInterrupt interrupt;
     ChainstateManager chainman{interrupt, chainman_opts, blockman_opts};
 
-    node::CacheSizes cache_sizes;
-    cache_sizes.block_tree_db = 2 << 20;
-    cache_sizes.coins_db = 2 << 22;
-    cache_sizes.coins = (450 << 20) - (2 << 20) - (2 << 22);
+    kernel::CacheSizes cache_sizes{MiBToBytes(DEFAULT_KERNEL_CACHE)};
     node::ChainstateLoadOptions options;
     auto [status, error] = node::LoadChainstate(chainman, cache_sizes, options);
     if (status != node::ChainstateLoadStatus::SUCCESS) {

src/init.cpp

@@ -32,6 +32,7 @@
 #include <interfaces/ipc.h>
 #include <interfaces/mining.h>
 #include <interfaces/node.h>
+#include <kernel/caches.h>
 #include <kernel/context.h>
 #include <key.h>
 #include <logging.h>
@@ -119,9 +120,10 @@ using common::AmountErrMsg;
 using common::InvalidPortErrMsg;
 using common::ResolveErrMsg;
 
+using kernel::CacheSizes;
+
 using node::ApplyArgsManOptions;
 using node::BlockManager;
-using node::CacheSizes;
 using node::CalculateCacheSizes;
 using node::ChainstateLoadResult;
 using node::ChainstateLoadStatus;
@@ -487,7 +489,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc)
     argsman.AddArg("-conf=<file>", strprintf("Specify path to read-only configuration file. Relative paths will be prefixed by datadir location (only useable from command line, not configuration file) (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS);
     argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS);
-    argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (minimum %d, default: %d). Make sure you have enough RAM. In addition, unused memory allocated to the mempool is shared with this cache (see -maxmempool).", nMinDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (minimum %d, default: %d). Make sure you have enough RAM. In addition, unused memory allocated to the mempool is shared with this cache (see -maxmempool).", MIN_DB_CACHE, DEFAULT_DB_CACHE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
@@ -1603,18 +1605,22 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
     ReadNotificationArgs(args, kernel_notifications);
 
     // cache size calculations
-    CacheSizes cache_sizes = CalculateCacheSizes(args, g_enabled_filter_types.size());
+    const auto cache_sizes = CalculateCacheSizes(args, g_enabled_filter_types.size());
+    if (!cache_sizes) {
+        return InitError(_("Failed to calculate cache sizes. Try lowering the -dbcache value."));
+    }
+    auto [index_cache_sizes, kernel_cache_sizes] = *cache_sizes;
 
-    LogPrintf("Cache configuration:\n");
-    LogPrintf("* Using %.1f MiB for block index database\n", cache_sizes.block_tree_db * (1.0 / 1024 / 1024));
+    LogInfo("Cache configuration:");
+    LogInfo("* Using %.1f MiB for block index database", kernel_cache_sizes.block_tree_db * (1.0 / 1024 / 1024));
     if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
-        LogPrintf("* Using %.1f MiB for transaction index database\n", cache_sizes.tx_index * (1.0 / 1024 / 1024));
+        LogInfo("* Using %.1f MiB for transaction index database", index_cache_sizes.tx_index * (1.0 / 1024 / 1024));
     }
     for (BlockFilterType filter_type : g_enabled_filter_types) {
-        LogPrintf("* Using %.1f MiB for %s block filter index database\n",
-                  cache_sizes.filter_index * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type));
+        LogInfo("* Using %.1f MiB for %s block filter index database",
+                  index_cache_sizes.filter_index * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type));
     }
-    LogPrintf("* Using %.1f MiB for chain state database\n", cache_sizes.coins_db * (1.0 / 1024 / 1024));
+    LogInfo("* Using %.1f MiB for chain state database", kernel_cache_sizes.coins_db * (1.0 / 1024 / 1024));
 
     assert(!node.mempool);
     assert(!node.chainman);
@@ -1627,7 +1633,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
         node,
         do_reindex,
         do_reindex_chainstate,
-        cache_sizes,
+        kernel_cache_sizes,
         args);
     if (status == ChainstateLoadStatus::FAILURE && !do_reindex && !ShutdownRequested(node)) {
         // suggest a reindex
@@ -1646,7 +1652,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
             node,
             do_reindex,
             do_reindex_chainstate,
-            cache_sizes,
+            kernel_cache_sizes,
             args);
     }
     if (status != ChainstateLoadStatus::SUCCESS && status != ChainstateLoadStatus::INTERRUPTED) {
@@ -1672,12 +1678,12 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
     // ********************************************************* Step 8: start indexers
 
     if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
-        g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, do_reindex);
+        g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), index_cache_sizes.tx_index, false, do_reindex);
         node.indexes.emplace_back(g_txindex.get());
     }
 
     for (const auto& filter_type : g_enabled_filter_types) {
-        InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, cache_sizes.filter_index, false, do_reindex);
+        InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, index_cache_sizes.filter_index, false, do_reindex);
         node.indexes.emplace_back(GetBlockFilterIndex(filter_type));
     }
 

src/kernel/caches.h

@@ -0,0 +1,48 @@
+// Copyright (c) 2024-present The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_KERNEL_CACHES_H
+#define BITCOIN_KERNEL_CACHES_H
+
+#include <util/check.h>
+
+#include <algorithm>
+#include <bit>
+#include <cstdint>
+#include <limits>
+
+//! Suggested default amount of cache reserved for the kernel (MiB)
+static constexpr int64_t DEFAULT_KERNEL_CACHE{450};
+//! Max memory allocated to block tree DB specific cache (MiB)
+static constexpr int64_t MAX_BLOCK_DB_CACHE{2};
+//! Max memory allocated to coin DB specific cache (MiB)
+static constexpr int64_t MAX_COINS_DB_CACHE{8};
+
+//! Guard against truncation of values before converting.
+constexpr size_t MiBToBytes(int64_t mib)
+{
+    Assert(std::countl_zero(static_cast<uint64_t>(mib)) >= 21); // Ensure sign bit is unset + enough zeros to shift.
+    const int64_t bytes{mib << 20};
+    Assert(static_cast<uint64_t>(bytes) <= std::numeric_limits<size_t>::max());
+    return static_cast<size_t>(bytes);
+}
+
+namespace kernel {
+struct CacheSizes {
+    size_t block_tree_db;
+    size_t coins_db;
+    size_t coins;
+
+    CacheSizes(size_t total_cache)
+    {
+        block_tree_db = std::min(total_cache / 8, MiBToBytes(MAX_BLOCK_DB_CACHE));
+        total_cache -= block_tree_db;
+        coins_db = std::min(total_cache / 2, MiBToBytes(MAX_COINS_DB_CACHE));
+        total_cache -= coins_db;
+        coins = total_cache; // the rest goes to the coins cache
+    }
+};
+} // namespace kernel
+
+#endif // BITCOIN_KERNEL_CACHES_H

src/node/caches.cpp

@@ -6,28 +6,42 @@
 
 #include <common/args.h>
 #include <index/txindex.h>
-#include <txdb.h>
+#include <kernel/caches.h>
+#include <logging.h>
+
+#include <algorithm>
+#include <optional>
+#include <string>
+
+// Unlike for the UTXO database, for the txindex scenario the leveldb cache make
+// a meaningful difference: https://github.com/bitcoin/bitcoin/pull/8273#issuecomment-229601991
+//! Max memory allocated to tx index DB specific cache in MiB.
+static constexpr int64_t MAX_TX_INDEX_CACHE{1024};
+//! Max memory allocated to all block filter index caches combined in MiB.
+static constexpr int64_t MAX_FILTER_INDEX_CACHE{1024};
 
 namespace node {
-CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes)
+std::optional<CacheSizes> CalculateCacheSizes(const ArgsManager& args, size_t n_indexes)
 {
-    int64_t nTotalCache = (args.GetIntArg("-dbcache", nDefaultDbCache) << 20);
-    nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache
-    CacheSizes sizes;
-    sizes.block_tree_db = std::min(nTotalCache / 8, nMaxBlockDBCache << 20);
-    nTotalCache -= sizes.block_tree_db;
-    sizes.tx_index = std::min(nTotalCache / 8, args.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0);
-    nTotalCache -= sizes.tx_index;
-    sizes.filter_index = 0;
-    if (n_indexes > 0) {
-        int64_t max_cache = std::min(nTotalCache / 8, max_filter_index_cache << 20);
-        sizes.filter_index = max_cache / n_indexes;
-        nTotalCache -= sizes.filter_index * n_indexes;
+    int64_t db_cache = args.GetIntArg("-dbcache", DEFAULT_DB_CACHE);
+    if (static_cast<uint64_t>(db_cache << 20) > std::numeric_limits<size_t>::max()) {
+        LogWarning("Cannot allocate more than %d MiB in total for db caches.", static_cast<double>(std::numeric_limits<size_t>::max()) * (1.0 / 1024 / 1024));
+        return std::nullopt;
     }
-    sizes.coins_db = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
-    sizes.coins_db = std::min(sizes.coins_db, nMaxCoinsDBCache << 20); // cap total coins db cache
-    nTotalCache -= sizes.coins_db;
-    sizes.coins = nTotalCache; // the rest goes to in-memory cache
-    return sizes;
+
+    // negative values are permitted, but interpreted as zero.
+    db_cache = std::max(int64_t{0}, db_cache);
+    size_t total_cache = std::max(MiBToBytes(db_cache), MiBToBytes(MIN_DB_CACHE));
+
+    IndexCacheSizes index_sizes;
+    index_sizes.tx_index = std::min(total_cache / 8, args.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? MiBToBytes(MAX_TX_INDEX_CACHE) : 0);
+    total_cache -= index_sizes.tx_index;
+    index_sizes.filter_index = 0;
+    if (n_indexes > 0) {
+        int64_t max_cache = std::min(total_cache / 8, MiBToBytes(MAX_FILTER_INDEX_CACHE));
+        index_sizes.filter_index = max_cache / n_indexes;
+        total_cache -= index_sizes.filter_index * n_indexes;
+    }
+    return {{index_sizes, kernel::CacheSizes{total_cache}}};
 }
 } // namespace node

src/node/caches.h

@@ -5,20 +5,29 @@
 #ifndef BITCOIN_NODE_CACHES_H
 #define BITCOIN_NODE_CACHES_H
 
+#include <kernel/caches.h>
+
 #include <cstddef>
 #include <cstdint>
+#include <optional>
 
 class ArgsManager;
 
+//! min. -dbcache (MiB)
+static constexpr int64_t MIN_DB_CACHE{4};
+//! -dbcache default (MiB)
+static constexpr int64_t DEFAULT_DB_CACHE{DEFAULT_KERNEL_CACHE};
+
 namespace node {
-struct CacheSizes {
-    int64_t block_tree_db;
-    int64_t coins_db;
-    int64_t coins;
-    int64_t tx_index;
-    int64_t filter_index;
+struct IndexCacheSizes {
+    size_t tx_index;
+    size_t filter_index;
 };
-CacheSizes CalculateCacheSizes(const ArgsManager& args, size_t n_indexes = 0);
+struct CacheSizes {
+    IndexCacheSizes index;
+    kernel::CacheSizes kernel;
+};
+std::optional<CacheSizes> CalculateCacheSizes(const ArgsManager& args, size_t n_indexes = 0);
 } // namespace node
 
 #endif // BITCOIN_NODE_CACHES_H

src/node/chainstate.cpp

@@ -8,9 +8,9 @@
 #include <chain.h>
 #include <coins.h>
 #include <consensus/params.h>
+#include <kernel/caches.h>
 #include <logging.h>
 #include <node/blockstorage.h>
-#include <node/caches.h>
 #include <sync.h>
 #include <threadsafety.h>
 #include <tinyformat.h>
@@ -29,6 +29,8 @@
 #include <memory>
 #include <vector>
 
+using kernel::CacheSizes;
+
 namespace node {
 // Complete initialization of chainstates after the initial call has been made
 // to ChainstateManager::InitializeChainstate().
@@ -44,7 +46,7 @@ static ChainstateLoadResult CompleteChainstateInitialization(
     try {
         pblocktree = std::make_unique<BlockTreeDB>(DBParams{
             .path = chainman.m_options.datadir / "blocks" / "index",
-            .cache_bytes = static_cast<size_t>(cache_sizes.block_tree_db),
+            .cache_bytes = cache_sizes.block_tree_db,
             .memory_only = options.block_tree_db_in_memory,
             .wipe_data = options.wipe_block_tree_db,
             .options = chainman.m_options.block_tree_db});

src/node/chainstate.h

@@ -14,9 +14,11 @@
 
 class CTxMemPool;
 
-namespace node {
-
+namespace kernel {
 struct CacheSizes;
+} // namespace kernel
+
+namespace node {
 
 struct ChainstateLoadOptions {
     CTxMemPool* mempool{nullptr};
@@ -69,7 +71,7 @@ using ChainstateLoadResult = std::tuple<ChainstateLoadStatus, bilingual_str>;
  *
  *  LoadChainstate returns a (status code, error string) tuple.
  */
-ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSizes& cache_sizes,
+ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const kernel::CacheSizes& cache_sizes,
                                     const ChainstateLoadOptions& options);
 ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const ChainstateLoadOptions& options);
 } // namespace node

src/qt/optionsdialog.cpp

@@ -15,9 +15,9 @@
 
 #include <common/system.h>
 #include <interfaces/node.h>
-#include <node/chainstatemanager_args.h>
 #include <netbase.h>
-#include <txdb.h>
+#include <node/caches.h>
+#include <node/chainstatemanager_args.h>
 #include <util/strencodings.h>
 
 #include <chrono>
@@ -95,7 +95,7 @@ OptionsDialog::OptionsDialog(QWidget* parent, bool enableWallet)
     ui->verticalLayout->setStretchFactor(ui->tabWidget, 1);
 
     /* Main elements init */
-    ui->databaseCache->setRange(nMinDbCache, std::numeric_limits<int>::max());
+    ui->databaseCache->setRange(MIN_DB_CACHE, std::numeric_limits<int>::max());
     ui->threadsScriptVerif->setMinimum(-GetNumCores());
     ui->threadsScriptVerif->setMaximum(MAX_SCRIPTCHECK_THREADS);
     ui->pruneWarning->setVisible(false);

src/qt/optionsmodel.cpp

@@ -15,8 +15,8 @@
 #include <mapport.h>
 #include <net.h>
 #include <netbase.h>
+#include <node/caches.h>
 #include <node/chainstatemanager_args.h>
-#include <txdb.h> // for -dbcache defaults
 #include <util/string.h>
 #include <validation.h>    // For DEFAULT_SCRIPTCHECK_THREADS
 #include <wallet/wallet.h> // For DEFAULT_SPEND_ZEROCONF_CHANGE
@@ -470,7 +470,7 @@ QVariant OptionsModel::getOption(OptionID option, const std::string& suffix) con
                suffix.empty()          ? getOption(option, "-prev") :
                                          DEFAULT_PRUNE_TARGET_GB;
     case DatabaseCache:
-        return qlonglong(SettingToInt(setting(), nDefaultDbCache));
+        return qlonglong(SettingToInt(setting(), DEFAULT_DB_CACHE));
     case ThreadsScriptVerif:
         return qlonglong(SettingToInt(setting(), DEFAULT_SCRIPTCHECK_THREADS));
     case Listen:
@@ -733,7 +733,7 @@ void OptionsModel::checkAndMigrate()
         // see https://github.com/bitcoin/bitcoin/pull/8273
         // force people to upgrade to the new value if they are using 100MB
         if (settingsVersion < 130000 && settings.contains("nDatabaseCache") && settings.value("nDatabaseCache").toLongLong() == 100)
-            settings.setValue("nDatabaseCache", (qint64)nDefaultDbCache);
+            settings.setValue("nDatabaseCache", (qint64)DEFAULT_DB_CACHE);
 
         settings.setValue(strSettingsVersionKey, CLIENT_VERSION);
     }

src/test/util/setup_common.cpp

@@ -66,7 +66,6 @@ using kernel::BlockTreeDB;
 using node::ApplyArgsManOptions;
 using node::BlockAssembler;
 using node::BlockManager;
-using node::CalculateCacheSizes;
 using node::KernelNotifications;
 using node::LoadChainstate;
 using node::RegenerateCommitments;
@@ -229,8 +228,6 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts)
     Assert(error.empty());
     m_node.warnings = std::make_unique<node::Warnings>();
 
-    m_cache_sizes = CalculateCacheSizes(m_args);
-
     m_node.notifications = std::make_unique<KernelNotifications>(Assert(m_node.shutdown_request), m_node.exit_status, *Assert(m_node.warnings));
 
     m_make_chainman = [this, &chainparams, opts] {
@@ -256,7 +253,7 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts)
         LOCK(m_node.chainman->GetMutex());
         m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<BlockTreeDB>(DBParams{
             .path = m_args.GetDataDirNet() / "blocks" / "index",
-            .cache_bytes = static_cast<size_t>(m_cache_sizes.block_tree_db),
+            .cache_bytes = m_kernel_cache_sizes.block_tree_db,
             .memory_only = true,
         });
     };
@@ -292,7 +289,7 @@ void ChainTestingSetup::LoadVerifyActivateChainstate()
     options.check_blocks = m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS);
     options.check_level = m_args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL);
     options.require_full_verification = m_args.IsArgSet("-checkblocks") || m_args.IsArgSet("-checklevel");
-    auto [status, error] = LoadChainstate(chainman, m_cache_sizes, options);
+    auto [status, error] = LoadChainstate(chainman, m_kernel_cache_sizes, options);
     assert(status == node::ChainstateLoadStatus::SUCCESS);
 
     std::tie(status, error) = VerifyLoadedChainstate(chainman, options);

src/test/util/setup_common.h

@@ -6,6 +6,7 @@
 #define BITCOIN_TEST_UTIL_SETUP_COMMON_H
 
 #include <common/args.h> // IWYU pragma: export
+#include <kernel/caches.h>
 #include <kernel/context.h>
 #include <key.h>
 #include <node/caches.h>
@@ -103,7 +104,7 @@ struct BasicTestingSetup {
  * initialization behaviour.
  */
 struct ChainTestingSetup : public BasicTestingSetup {
-    node::CacheSizes m_cache_sizes{};
+    kernel::CacheSizes m_kernel_cache_sizes{node::CalculateCacheSizes(m_args).value().kernel};
     bool m_coins_db_in_memory{true};
     bool m_block_tree_db_in_memory{true};
     std::function<void()> m_make_chainman{};

src/txdb.h

@@ -21,22 +21,8 @@
 class COutPoint;
 class uint256;
 
-//! -dbcache default (MiB)
-static const int64_t nDefaultDbCache = 450;
 //! -dbbatchsize default (bytes)
 static const int64_t nDefaultDbBatchSize = 16 << 20;
-//! min. -dbcache (MiB)
-static const int64_t nMinDbCache = 4;
-//! Max memory allocated to block tree DB specific cache, if no -txindex (MiB)
-static const int64_t nMaxBlockDBCache = 2;
-//! Max memory allocated to block tree DB specific cache, if -txindex (MiB)
-// Unlike for the UTXO database, for the txindex scenario the leveldb cache make
-// a meaningful difference: https://github.com/bitcoin/bitcoin/pull/8273#issuecomment-229601991
-static const int64_t nMaxTxIndexCache = 1024;
-//! Max memory allocated to all block filter index caches combined in MiB.
-static const int64_t max_filter_index_cache = 1024;
-//! Max memory allocated to coin DB specific cache (MiB)
-static const int64_t nMaxCoinsDBCache = 8;
 
 //! User-controlled performance and debug options.
 struct CoinsViewOptions {

src/validation.h

@@ -1064,11 +1064,11 @@ public:
 
     //! The total number of bytes available for us to use across all in-memory
     //! coins caches. This will be split somehow across chainstates.
-    int64_t m_total_coinstip_cache{0};
+    size_t m_total_coinstip_cache{0};
     //
     //! The total number of bytes available for us to use across all leveldb
     //! coins databases. This will be split somehow across chainstates.
-    int64_t m_total_coinsdb_cache{0};
+    size_t m_total_coinsdb_cache{0};
 
     //! Instantiate a new chainstate.
     //!