This commit is contained in:
l0rinc 2025-01-08 19:18:50 +01:00 committed by GitHub
commit 5707851c3a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 183 additions and 165 deletions

View file

@ -44,7 +44,7 @@ add_executable(bench_bitcoin
pool.cpp
prevector.cpp
random.cpp
readblock.cpp
readwriteblock.cpp
rollingbloom.cpp
rpc_blockchain.cpp
rpc_mempool.cpp

View file

@ -1,60 +0,0 @@
// Copyright (c) 2023 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <bench/bench.h>
#include <bench/data/block413567.raw.h>
#include <flatfile.h>
#include <node/blockstorage.h>
#include <primitives/block.h>
#include <primitives/transaction.h>
#include <serialize.h>
#include <span.h>
#include <streams.h>
#include <test/util/setup_common.h>
#include <validation.h>
#include <cassert>
#include <cstdint>
#include <memory>
#include <vector>
static FlatFilePos WriteBlockToDisk(ChainstateManager& chainman)
{
DataStream stream{benchmark::data::block413567};
CBlock block;
stream >> TX_WITH_WITNESS(block);
return chainman.m_blockman.SaveBlockToDisk(block, 0);
}
static void ReadBlockFromDiskTest(benchmark::Bench& bench)
{
const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)};
ChainstateManager& chainman{*testing_setup->m_node.chainman};
CBlock block;
const auto pos{WriteBlockToDisk(chainman)};
bench.run([&] {
const auto success{chainman.m_blockman.ReadBlockFromDisk(block, pos)};
assert(success);
});
}
static void ReadRawBlockFromDiskTest(benchmark::Bench& bench)
{
const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)};
ChainstateManager& chainman{*testing_setup->m_node.chainman};
std::vector<uint8_t> block_data;
const auto pos{WriteBlockToDisk(chainman)};
bench.run([&] {
const auto success{chainman.m_blockman.ReadRawBlockFromDisk(block_data, pos)};
assert(success);
});
}
BENCHMARK(ReadBlockFromDiskTest, benchmark::PriorityLevel::HIGH);
BENCHMARK(ReadRawBlockFromDiskTest, benchmark::PriorityLevel::HIGH);

View file

@ -0,0 +1,79 @@
// Copyright (c) 2023 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <bench/bench.h>
#include <bench/data/block413567.raw.h>
#include <flatfile.h>
#include <node/blockstorage.h>
#include <primitives/block.h>
#include <primitives/transaction.h>
#include <serialize.h>
#include <span.h>
#include <streams.h>
#include <test/util/setup_common.h>
#include <validation.h>
#include <cassert>
#include <cstdint>
#include <memory>
#include <vector>
CBlock CreateTestBlock()
{
DataStream stream{benchmark::data::block413567};
CBlock block;
stream >> TX_WITH_WITNESS(block);
return block;
}
static void GetSerializeSizeBench(benchmark::Bench& bench)
{
const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)};
const CBlock block{CreateTestBlock()};
bench.run([&] {
const uint32_t block_size{static_cast<uint32_t>(GetSerializeSize(TX_WITH_WITNESS(block)))};
assert(block_size == benchmark::data::block413567.size());
});
}
static void SaveBlockToDiskBench(benchmark::Bench& bench)
{
const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)};
auto& blockman{testing_setup->m_node.chainman->m_blockman};
const CBlock block{CreateTestBlock()};
bench.run([&] {
const auto pos{blockman.SaveBlock(block, 413'567)};
assert(!pos.IsNull());
});
}
static void ReadBlockFromDiskBench(benchmark::Bench& bench)
{
const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)};
auto& blockman{testing_setup->m_node.chainman->m_blockman};
const auto pos{blockman.SaveBlock(CreateTestBlock(), 413'567)};
CBlock block;
bench.run([&] {
const auto success{blockman.ReadBlockFromDisk(block, pos)};
assert(success);
});
}
static void ReadRawBlockFromDiskBench(benchmark::Bench& bench)
{
const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)};
auto& blockman{testing_setup->m_node.chainman->m_blockman};
const auto pos{blockman.SaveBlock(CreateTestBlock(), 413'567)};
std::vector<uint8_t> block_data;
blockman.ReadRawBlockFromDisk(block_data, pos); // warmup
bench.run([&] {
const auto success{blockman.ReadRawBlockFromDisk(block_data, pos)};
assert(success);
});
}
BENCHMARK(GetSerializeSizeBench, benchmark::PriorityLevel::HIGH);
BENCHMARK(SaveBlockToDiskBench, benchmark::PriorityLevel::HIGH);
BENCHMARK(ReadBlockFromDiskBench, benchmark::PriorityLevel::HIGH);
BENCHMARK(ReadRawBlockFromDiskBench, benchmark::PriorityLevel::HIGH);

View file

@ -669,36 +669,12 @@ CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n)
return &m_blockfile_info.at(n);
}
bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const
{
// Open history file to append
AutoFile fileout{OpenUndoFile(pos)};
if (fileout.IsNull()) {
LogError("%s: OpenUndoFile failed\n", __func__);
return false;
}
// Write index header
unsigned int nSize = GetSerializeSize(blockundo);
fileout << GetParams().MessageStart() << nSize;
// Write undo data
long fileOutPos = fileout.tell();
pos.nPos = (unsigned int)fileOutPos;
fileout << blockundo;
// calculate & write checksum
HashWriter hasher{};
hasher << hashBlock;
hasher << blockundo;
fileout << hasher.GetHash();
return true;
}
bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const
{
const FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())};
FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())};
if (pos.nPos < BLOCK_SERIALIZATION_HEADER_SIZE) return false;
uint32_t undo_size;
pos.nPos -= sizeof undo_size;
// Open history file to read
AutoFile filein{OpenUndoFile(pos, true)};
@ -708,23 +684,29 @@ bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& in
}
// Read block
uint256 hashChecksum;
HashVerifier verifier{filein}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a
try {
filein >> undo_size;
if (undo_size > MAX_SIZE) throw std::runtime_error{strprintf("Refusing to read undo data of size: %d", undo_size)};
std::vector<uint8_t> mem(undo_size);
filein >> Span{mem};
SpanReader reader{mem};
HashVerifier verifier{reader}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a
verifier << index.pprev->GetBlockHash();
verifier >> blockundo;
uint256 hashChecksum;
filein >> hashChecksum;
if (hashChecksum != verifier.GetHash()) {
LogError("%s: Checksum mismatch at %s\n", __func__, pos.ToString());
return false;
}
} catch (const std::exception& e) {
LogError("%s: Deserialize or I/O error - %s at %s\n", __func__, e.what(), pos.ToString());
return false;
}
// Verify checksum
if (hashChecksum != verifier.GetHash()) {
LogError("%s: Checksum mismatch at %s\n", __func__, pos.ToString());
return false;
}
return true;
}
@ -963,62 +945,64 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP
return true;
}
bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const
{
// Open history file to append
AutoFile fileout{OpenBlockFile(pos)};
if (fileout.IsNull()) {
LogError("%s: OpenBlockFile failed\n", __func__);
return false;
}
// Write index header
unsigned int nSize = GetSerializeSize(TX_WITH_WITNESS(block));
fileout << GetParams().MessageStart() << nSize;
// Write block
long fileOutPos = fileout.tell();
pos.nPos = (unsigned int)fileOutPos;
fileout << TX_WITH_WITNESS(block);
return true;
}
bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
bool BlockManager::SaveBlockUndo(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
{
AssertLockHeld(::cs_main);
const BlockfileType type = BlockfileTypeForHeight(block.nHeight);
auto& cursor = *Assert(WITH_LOCK(cs_LastBlockFile, return m_blockfile_cursors[type]));
// Write undo information to disk
if (block.GetUndoPos().IsNull()) {
FlatFilePos _pos;
if (!FindUndoPos(state, block.nFile, _pos, ::GetSerializeSize(blockundo) + 40)) {
FlatFilePos pos;
const uint32_t blockundo_size{static_cast<uint32_t>(GetSerializeSize(blockundo))};
if (!FindUndoPos(state, block.nFile, pos, UNDO_DATA_DISK_OVERHEAD + blockundo_size)) {
LogError("%s: FindUndoPos failed\n", __func__);
return false;
}
if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash())) {
AutoFile fileout{m_undo_file_seq.Open(pos, false), {}}; // We'll obfuscate ourselves
if (fileout.IsNull()) {
LogError("%s: OpenUndoFile failed\n", __func__);
return FatalError(m_opts.notifications, state, _("Failed to write undo data."));
}
{
DataStream header;
header.reserve(BLOCK_SERIALIZATION_HEADER_SIZE);
header << GetParams().MessageStart() << blockundo_size;
util::Xor(header, m_xor_key, pos.nPos);
fileout.write(header);
}
pos.nPos += BLOCK_SERIALIZATION_HEADER_SIZE;
{
HashWriter hasher;
hasher << block.pprev->GetBlockHash();
hasher << blockundo;
DataStream undo_data;
undo_data.reserve(blockundo_size + sizeof(uint256));
undo_data << blockundo << hasher.GetHash();
util::Xor(undo_data, m_xor_key, pos.nPos);
fileout.write(undo_data);
}
// rev files are written in block height order, whereas blk files are written as blocks come in (often out of order)
// we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height
// in the block file info as below; note that this does not catch the case where the undo writes are keeping up
// with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in
// the FindNextBlockPos function
if (_pos.nFile < cursor.file_num && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) {
if (pos.nFile < cursor.file_num && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[pos.nFile].nHeightLast) {
// Do not propagate the return code, a failed flush here should not
// be an indication for a failed write. If it were propagated here,
// the caller would assume the undo data not to be written, when in
// fact it is. Note though, that a failed flush might leave the data
// file untrimmed.
if (!FlushUndoFile(_pos.nFile, true)) {
LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, "Failed to flush undo file %05i\n", _pos.nFile);
if (!FlushUndoFile(pos.nFile, true)) {
LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, "Failed to flush undo file %05i\n", pos.nFile);
}
} else if (_pos.nFile == cursor.file_num && block.nHeight > cursor.undo_height) {
} else if (pos.nFile == cursor.file_num && block.nHeight > cursor.undo_height) {
cursor.undo_height = block.nHeight;
}
// update nUndoPos in block index
block.nUndoPos = _pos.nPos;
block.nUndoPos = pos.nPos;
block.nStatus |= BLOCK_HAVE_UNDO;
m_dirty_blockindex.insert(&block);
}
@ -1026,20 +1010,26 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid
return true;
}
bool BlockManager::ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const
bool BlockManager::ReadBlockFromDisk(CBlock& block, FlatFilePos pos) const
{
block.SetNull();
// Open history file to read
if (pos.nPos < BLOCK_SERIALIZATION_HEADER_SIZE) return false;
uint32_t blk_size;
pos.nPos -= sizeof blk_size;
AutoFile filein{OpenBlockFile(pos, true)};
if (filein.IsNull()) {
LogError("%s: OpenBlockFile failed for %s\n", __func__, pos.ToString());
return false;
}
// Read block
try {
filein >> TX_WITH_WITNESS(block);
filein >> blk_size;
if (blk_size > MAX_SIZE) throw std::runtime_error{strprintf("Refusing to read block of size: %d", blk_size)};
std::vector<uint8_t> mem(blk_size);
filein >> Span{mem};
SpanReader(mem) >> TX_WITH_WITNESS(block);
} catch (const std::exception& e) {
LogError("%s: Deserialize or I/O error - %s at %s\n", __func__, e.what(), pos.ToString());
return false;
@ -1119,22 +1109,38 @@ bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatF
return true;
}
FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight)
FlatFilePos BlockManager::SaveBlock(const CBlock& block, int nHeight)
{
unsigned int nBlockSize = ::GetSerializeSize(TX_WITH_WITNESS(block));
// Account for the 4 magic message start bytes + the 4 length bytes (8 bytes total,
// defined as BLOCK_SERIALIZATION_HEADER_SIZE)
nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE);
FlatFilePos blockPos{FindNextBlockPos(nBlockSize, nHeight, block.GetBlockTime())};
if (blockPos.IsNull()) {
const uint32_t block_size{static_cast<uint32_t>(GetSerializeSize(TX_WITH_WITNESS(block)))};
FlatFilePos pos{FindNextBlockPos(BLOCK_SERIALIZATION_HEADER_SIZE + block_size, nHeight, block.GetBlockTime())};
if (pos.IsNull()) {
LogError("%s: FindNextBlockPos failed\n", __func__);
return FlatFilePos();
}
if (!WriteBlockToDisk(block, blockPos)) {
AutoFile fileout{m_block_file_seq.Open(pos, false), {}}; // We'll obfuscate ourselves
if (fileout.IsNull()) {
LogError("%s: OpenBlockFile failed\n", __func__);
m_opts.notifications.fatalError(_("Failed to write block."));
return FlatFilePos();
}
return blockPos;
{
DataStream header;
header.reserve(BLOCK_SERIALIZATION_HEADER_SIZE);
header << GetParams().MessageStart() << block_size;
util::Xor(header, m_xor_key, pos.nPos);
fileout.write(header);
}
pos.nPos += BLOCK_SERIALIZATION_HEADER_SIZE;
{
DataStream block_data;
block_data.reserve(block_size);
block_data << TX_WITH_WITNESS(block);
util::Xor(block_data, m_xor_key, pos.nPos);
fileout.write(block_data);
}
return pos;
}
static auto InitBlocksdirXorKey(const BlockManager::Options& opts)

View file

@ -74,8 +74,11 @@ static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB
/** The maximum size of a blk?????.dat file (since 0.8) */
static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
/** Size of header written by WriteBlockToDisk before a serialized CBlock */
static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = std::tuple_size_v<MessageStartChars> + sizeof(unsigned int);
/** Size of header written by SaveBlock before a serialized CBlock (8 bytes) */
static constexpr uint32_t BLOCK_SERIALIZATION_HEADER_SIZE = std::tuple_size_v<MessageStartChars> + sizeof(uint32_t);
/** Total overhead when writing undo data: header (8 bytes) plus checksum (32 bytes) */
static constexpr uint32_t UNDO_DATA_DISK_OVERHEAD = BLOCK_SERIALIZATION_HEADER_SIZE + uint256::size();
// Because validation code takes pointers to the map's CBlockIndex objects, if
// we ever switch to another associative container, we need to either use a
@ -161,7 +164,7 @@ private:
* blockfile info, and checks if there is enough disk space to save the block.
*
* The nAddSize argument passed to this function should include not just the size of the serialized CBlock, but also the size of
* separator fields which are written before it by WriteBlockToDisk (BLOCK_SERIALIZATION_HEADER_SIZE).
* separator fields (BLOCK_SERIALIZATION_HEADER_SIZE).
*/
[[nodiscard]] FlatFilePos FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime);
[[nodiscard]] bool FlushChainstateBlockFile(int tip_height);
@ -169,15 +172,6 @@ private:
AutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const;
/**
* Write a block to disk. The pos argument passed to this function is modified by this call. Before this call, it should
* point to an unused file location where separator fields will be written, followed by the serialized CBlock data.
* After this call, it will point to the beginning of the serialized CBlock data, after the separator fields
* (BLOCK_SERIALIZATION_HEADER_SIZE)
*/
bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const;
bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const;
/* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */
void FindFilesToPruneManual(
std::set<int>& setFilesToPrune,
@ -330,7 +324,7 @@ public:
/** Get block file info entry for one block file */
CBlockFileInfo* GetBlockFileInfo(size_t n);
bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
bool SaveBlockUndo(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
/** Store block on disk and update block file statistics.
@ -341,14 +335,13 @@ public:
* @returns in case of success, the position to which the block was written to
* in case of an error, an empty FlatFilePos
*/
FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight);
FlatFilePos SaveBlock(const CBlock& block, int nHeight);
/** Update blockfile info while processing a block during reindex. The block must be available on disk.
*
* @param[in] block the block being processed
* @param[in] nHeight the height of the block
* @param[in] pos the position of the serialized CBlock on disk. This is the position returned
* by WriteBlockToDisk pointing at the CBlock, not the separator fields before it
* @param[in] pos the position of the serialized CBlock on disk
*/
void UpdateBlockInfo(const CBlock& block, unsigned int nHeight, const FlatFilePos& pos);
@ -421,7 +414,7 @@ public:
void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const;
/** Functions for disk access for blocks */
bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const;
bool ReadBlockFromDisk(CBlock& block, FlatFilePos pos) const;
bool ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const;
bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos) const;

View file

@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos)
};
BlockManager blockman{*Assert(m_node.shutdown_signal), blockman_opts};
// simulate adding a genesis block normally
BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0).nPos, BLOCK_SERIALIZATION_HEADER_SIZE);
BOOST_CHECK_EQUAL(blockman.SaveBlock(params->GenesisBlock(), 0).nPos, BLOCK_SERIALIZATION_HEADER_SIZE);
// simulate what happens during reindex
// simulate a well-formed genesis block being found at offset 8 in the blk00000.dat file
// the block is found at offset 8 because there is an 8 byte serialization header
@ -49,7 +49,7 @@ BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos)
// this is a check to make sure that https://github.com/bitcoin/bitcoin/issues/21379 does not recur
// 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293
// add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301
FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1)};
FlatFilePos actual{blockman.SaveBlock(params->GenesisBlock(), 1)};
BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(TX_WITH_WITNESS(params->GenesisBlock())) + BLOCK_SERIALIZATION_HEADER_SIZE);
}
@ -158,10 +158,10 @@ BOOST_AUTO_TEST_CASE(blockmanager_flush_block_file)
BOOST_CHECK_EQUAL(blockman.CalculateCurrentUsage(), 0);
// Write the first block to a new location.
FlatFilePos pos1{blockman.SaveBlockToDisk(block1, /*nHeight=*/1)};
FlatFilePos pos1{blockman.SaveBlock(block1, /*nHeight=*/1)};
// Write second block
FlatFilePos pos2{blockman.SaveBlockToDisk(block2, /*nHeight=*/2)};
FlatFilePos pos2{blockman.SaveBlock(block2, /*nHeight=*/2)};
// Two blocks in the file
BOOST_CHECK_EQUAL(blockman.CalculateCurrentUsage(), (TEST_BLOCK_SIZE + BLOCK_SERIALIZATION_HEADER_SIZE) * 2);

View file

@ -2747,7 +2747,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state,
return true;
}
if (!m_blockman.WriteUndoDataForBlock(blockundo, state, *pindex)) {
if (!m_blockman.SaveBlockUndo(blockundo, state, *pindex)) {
return false;
}
@ -4564,7 +4564,7 @@ bool ChainstateManager::AcceptBlock(const std::shared_ptr<const CBlock>& pblock,
blockPos = *dbp;
m_blockman.UpdateBlockInfo(block, pindex->nHeight, blockPos);
} else {
blockPos = m_blockman.SaveBlockToDisk(block, pindex->nHeight);
blockPos = m_blockman.SaveBlock(block, pindex->nHeight);
if (blockPos.IsNull()) {
state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__));
return false;
@ -5062,7 +5062,7 @@ bool Chainstate::LoadGenesisBlock()
try {
const CBlock& block = params.GenesisBlock();
FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0)};
FlatFilePos blockPos{m_blockman.SaveBlock(block, 0)};
if (blockPos.IsNull()) {
LogError("%s: writing genesis block to disk failed\n", __func__);
return false;