phantom/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-01-09 11:27:28 -03:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge 898a07e2ab into 66aa6a47bd

Browse source
This commit is contained in:
l0rinc 2025-01-08 13:01:40 -05:00 committed by GitHub
commit 6aa09dff9d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 461 additions and 302 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/bench/CMakeLists.txt
View file

@ -44,7 +44,7 @@ add_executable(bench_bitcoin
pool.cpp
prevector.cpp
random.cpp
readblock.cpp
readwriteblock.cpp
rollingbloom.cpp
rpc_blockchain.cpp
rpc_mempool.cpp

60
src/bench/readblock.cpp
View file

@ -1,60 +0,0 @@
// Copyright (c) 2023 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <bench/bench.h>
#include <bench/data/block413567.raw.h>
#include <flatfile.h>
#include <node/blockstorage.h>
#include <primitives/block.h>
#include <primitives/transaction.h>
#include <serialize.h>
#include <span.h>
#include <streams.h>
#include <test/util/setup_common.h>
#include <validation.h>
#include <cassert>
#include <cstdint>
#include <memory>
#include <vector>
static FlatFilePos WriteBlockToDisk(ChainstateManager& chainman)
{
DataStream stream{benchmark::data::block413567};
CBlock block;
stream >> TX_WITH_WITNESS(block);
return chainman.m_blockman.SaveBlockToDisk(block, 0);
}
static void ReadBlockFromDiskTest(benchmark::Bench& bench)
{
const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)};
ChainstateManager& chainman{*testing_setup->m_node.chainman};
CBlock block;
const auto pos{WriteBlockToDisk(chainman)};
bench.run([&] {
const auto success{chainman.m_blockman.ReadBlockFromDisk(block, pos)};
assert(success);
});
}
static void ReadRawBlockFromDiskTest(benchmark::Bench& bench)
{
const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)};
ChainstateManager& chainman{*testing_setup->m_node.chainman};
std::vector<uint8_t> block_data;
const auto pos{WriteBlockToDisk(chainman)};
bench.run([&] {
const auto success{chainman.m_blockman.ReadRawBlockFromDisk(block_data, pos)};
assert(success);
});
}
BENCHMARK(ReadBlockFromDiskTest, benchmark::PriorityLevel::HIGH);
BENCHMARK(ReadRawBlockFromDiskTest, benchmark::PriorityLevel::HIGH);

79
src/bench/readwriteblock.cpp Normal file
View file

@ -0,0 +1,79 @@
// Copyright (c) 2023 The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <bench/bench.h>
#include <bench/data/block413567.raw.h>
#include <flatfile.h>
#include <node/blockstorage.h>
#include <primitives/block.h>
#include <primitives/transaction.h>
#include <serialize.h>
#include <span.h>
#include <streams.h>
#include <test/util/setup_common.h>
#include <validation.h>
#include <cassert>
#include <cstdint>
#include <memory>
#include <vector>
CBlock CreateTestBlock()
{
DataStream stream{benchmark::data::block413567};
CBlock block;
stream >> TX_WITH_WITNESS(block);
return block;
}
static void GetSerializeSizeBench(benchmark::Bench& bench)
{
const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)};
const CBlock block{CreateTestBlock()};
bench.run([&] {
const uint32_t block_size{static_cast<uint32_t>(GetSerializeSize(TX_WITH_WITNESS(block)))};
assert(block_size == benchmark::data::block413567.size());
});
}
static void SaveBlockToDiskBench(benchmark::Bench& bench)
{
const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)};
auto& blockman{testing_setup->m_node.chainman->m_blockman};
const CBlock block{CreateTestBlock()};
bench.run([&] {
const auto pos{blockman.SaveBlock(block, 413'567)};
assert(!pos.IsNull());
});
}
static void ReadBlockFromDiskBench(benchmark::Bench& bench)
{
const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)};
auto& blockman{testing_setup->m_node.chainman->m_blockman};
const auto pos{blockman.SaveBlock(CreateTestBlock(), 413'567)};
CBlock block;
bench.run([&] {
const auto success{blockman.ReadBlockFromDisk(block, pos)};
assert(success);
});
}
static void ReadRawBlockFromDiskBench(benchmark::Bench& bench)
{
const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)};
auto& blockman{testing_setup->m_node.chainman->m_blockman};
const auto pos{blockman.SaveBlock(CreateTestBlock(), 413'567)};
std::vector<uint8_t> block_data;
blockman.ReadRawBlockFromDisk(block_data, pos); // warmup
bench.run([&] {
const auto success{blockman.ReadRawBlockFromDisk(block_data, pos)};
assert(success);
});
}
BENCHMARK(GetSerializeSizeBench, benchmark::PriorityLevel::HIGH);
BENCHMARK(SaveBlockToDiskBench, benchmark::PriorityLevel::HIGH);
BENCHMARK(ReadBlockFromDiskBench, benchmark::PriorityLevel::HIGH);
BENCHMARK(ReadRawBlockFromDiskBench, benchmark::PriorityLevel::HIGH);

16
src/bench/xor.cpp
View file

@ -7,17 +7,23 @@
#include <span.h>
#include <streams.h>
#include <cmath>
#include <cstddef>
#include <map>
#include <vector>
static void Xor(benchmark::Bench& bench)
{
FastRandomContext frc{/*fDeterministic=*/true};
auto data{frc.randbytes<std::byte>(1024)};
auto key{frc.randbytes<std::byte>(31)};
FastRandomContext rng{/*fDeterministic=*/true};
auto test_data{rng.randbytes<std::byte>(1 << 20)};
bench.batch(data.size()).unit("byte").run([&] {
util::Xor(data, key);
const Obfuscation obfuscation{rng.rand64()};
assert(obfuscation);
size_t offset{0};
bench.batch(test_data.size()).unit("byte").run([&] {
obfuscation(test_data, offset++);
ankerl::nanobench::doNotOptimizeAway(test_data);
});
}

53
src/dbwrapper.cpp
View file

@ -171,7 +171,7 @@ void CDBBatch::Clear()
void CDBBatch::WriteImpl(Span<const std::byte> key, DataStream& ssValue)
{
leveldb::Slice slKey(CharCast(key.data()), key.size());
ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
dbwrapper_private::GetObfuscation(parent)(ssValue);
leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size());
m_impl_batch->batch.Put(slKey, slValue);
// LevelDB serializes writes as:
@ -220,7 +220,11 @@ struct LevelDBContext {
};
CDBWrapper::CDBWrapper(const DBParams& params)
: m_db_context{std::make_unique<LevelDBContext>()}, m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only}
: m_db_context{std::make_unique<LevelDBContext>()},
m_name{fs::PathToString(params.path.stem())},
m_obfuscation{0},
m_path{params.path},
m_is_memory{params.memory_only}
{
DBContext().penv = nullptr;
DBContext().readoptions.verify_checksums = true;
@ -255,24 +259,23 @@ CDBWrapper::CDBWrapper(const DBParams& params)
LogPrintf("Finished database compaction of %s\n", fs::PathToString(params.path));
}
// The base-case obfuscation key, which is a noop.
obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
if (!key_exists && params.obfuscate && IsEmpty()) {
// Initialize non-degenerate obfuscation if it won't upset
// existing, non-obfuscated data.
std::vector<unsigned char> new_key = CreateObfuscateKey();
m_obfuscation = 0; // Needed for unobfuscated Read
std::vector<unsigned char> obfuscate_key_vector(Obfuscation::SIZE_BYTES, '\000');
const bool key_missing = !Read(OBFUSCATE_KEY_KEY, obfuscate_key_vector);
if (key_missing && params.obfuscate && IsEmpty()) {
// Initialize non-degenerate obfuscation if it won't upset existing, non-obfuscated data.
std::vector<uint8_t> new_key(Obfuscation::SIZE_BYTES);
GetRandBytes(new_key);
// Write `new_key` so we don't obfuscate the key with itself
Write(OBFUSCATE_KEY_KEY, new_key);
obfuscate_key = new_key;
obfuscate_key_vector = new_key;
LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key));
LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key_vector));
}
LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key));
LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key_vector));
m_obfuscation = obfuscate_key_vector;
obfuscate_key_vector.clear();
}
CDBWrapper::~CDBWrapper()
@ -323,19 +326,6 @@ size_t CDBWrapper::DynamicMemoryUsage() const
// past the null-terminator.
const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
/**
* Returns a string (consisting of 8 random bytes) suitable for use as an
* obfuscating XOR key.
*/
std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
{
std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES);
GetRandBytes(ret);
return ret;
}
std::optional<std::string> CDBWrapper::ReadImpl(Span<const std::byte> key) const
{
leveldb::Slice slKey(CharCast(key.data()), key.size());
@ -418,10 +408,5 @@ void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); }
void CDBIterator::Next() { m_impl_iter->iter->Next(); }
namespace dbwrapper_private {
const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w)
{
return w.obfuscate_key;
}
Obfuscation GetObfuscation(const CDBWrapper& w) { return w.m_obfuscation; }
} // namespace dbwrapper_private

16
src/dbwrapper.h
View file

@ -63,8 +63,7 @@ namespace dbwrapper_private {
* Database obfuscation should be considered an implementation detail of the
* specific database.
*/
const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w);
Obfuscation GetObfuscation(const CDBWrapper&);
}; // namespace dbwrapper_private
bool DestroyDB(const std::string& path_str);
@ -168,7 +167,7 @@ public:
template<typename V> bool GetValue(V& value) {
try {
DataStream ssValue{GetValueImpl()};
ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
dbwrapper_private::GetObfuscation(parent)(ssValue);
ssValue >> value;
} catch (const std::exception&) {
return false;
@ -181,7 +180,7 @@ struct LevelDBContext;
class CDBWrapper
{
friend const std::vector<unsigned char>& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w);
friend Obfuscation dbwrapper_private::GetObfuscation(const CDBWrapper&);
private:
//! holds all leveldb-specific fields of this class
std::unique_ptr<LevelDBContext> m_db_context;
@ -190,16 +189,11 @@ private:
std::string m_name;
//! a key used for optional XOR-obfuscation of the database
std::vector<unsigned char> obfuscate_key;
Obfuscation m_obfuscation;
//! the key under which the obfuscation key is stored
static const std::string OBFUSCATE_KEY_KEY;
//! the length of the obfuscate key in number of bytes
static const unsigned int OBFUSCATE_KEY_NUM_BYTES;
std::vector<unsigned char> CreateObfuscateKey() const;
//! path to filesystem storage
const fs::path m_path;
@ -230,7 +224,7 @@ public:
}
try {
DataStream ssValue{MakeByteSpan(*strValue)};
ssValue.Xor(obfuscate_key);
m_obfuscation(ssValue);
ssValue >> value;
} catch (const std::exception&) {
return false;

176
src/node/blockstorage.cpp
View file

@ -669,36 +669,12 @@ CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n)
return &m_blockfile_info.at(n);
}
bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const
{
// Open history file to append
AutoFile fileout{OpenUndoFile(pos)};
if (fileout.IsNull()) {
LogError("%s: OpenUndoFile failed\n", __func__);
return false;
}
// Write index header
unsigned int nSize = GetSerializeSize(blockundo);
fileout << GetParams().MessageStart() << nSize;
// Write undo data
long fileOutPos = fileout.tell();
pos.nPos = (unsigned int)fileOutPos;
fileout << blockundo;
// calculate & write checksum
HashWriter hasher{};
hasher << hashBlock;
hasher << blockundo;
fileout << hasher.GetHash();
return true;
}
bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const
{
const FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())};
FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())};
if (pos.nPos < BLOCK_SERIALIZATION_HEADER_SIZE) return false;
uint32_t undo_size;
pos.nPos -= sizeof undo_size;
// Open history file to read
AutoFile filein{OpenUndoFile(pos, true)};
@ -708,23 +684,29 @@ bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& in
}
// Read block
uint256 hashChecksum;
HashVerifier verifier{filein}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a
try {
filein >> undo_size;
if (undo_size > MAX_SIZE) throw std::runtime_error{strprintf("Refusing to read undo data of size: %d", undo_size)};
std::vector<uint8_t> mem(undo_size);
filein >> Span{mem};
SpanReader reader{mem};
HashVerifier verifier{reader}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a
verifier << index.pprev->GetBlockHash();
verifier >> blockundo;
uint256 hashChecksum;
filein >> hashChecksum;
if (hashChecksum != verifier.GetHash()) {
LogError("%s: Checksum mismatch at %s\n", __func__, pos.ToString());
return false;
}
} catch (const std::exception& e) {
LogError("%s: Deserialize or I/O error - %s at %s\n", __func__, e.what(), pos.ToString());
return false;
}
// Verify checksum
if (hashChecksum != verifier.GetHash()) {
LogError("%s: Checksum mismatch at %s\n", __func__, pos.ToString());
return false;
}
return true;
}
@ -815,13 +797,13 @@ void BlockManager::UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const
AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const
{
return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_xor_key};
return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_obfuscation};
}
/** Open an undo file (rev?????.dat) */
AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const
{
return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_xor_key};
return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_obfuscation};
}
fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const
@ -963,62 +945,64 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP
return true;
}
bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const
{
// Open history file to append
AutoFile fileout{OpenBlockFile(pos)};
if (fileout.IsNull()) {
LogError("%s: OpenBlockFile failed\n", __func__);
return false;
}
// Write index header
unsigned int nSize = GetSerializeSize(TX_WITH_WITNESS(block));
fileout << GetParams().MessageStart() << nSize;
// Write block
long fileOutPos = fileout.tell();
pos.nPos = (unsigned int)fileOutPos;
fileout << TX_WITH_WITNESS(block);
return true;
}
bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
bool BlockManager::SaveBlockUndo(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
{
AssertLockHeld(::cs_main);
const BlockfileType type = BlockfileTypeForHeight(block.nHeight);
auto& cursor = *Assert(WITH_LOCK(cs_LastBlockFile, return m_blockfile_cursors[type]));
// Write undo information to disk
if (block.GetUndoPos().IsNull()) {
FlatFilePos _pos;
if (!FindUndoPos(state, block.nFile, _pos, ::GetSerializeSize(blockundo) + 40)) {
FlatFilePos pos;
const uint32_t blockundo_size{static_cast<uint32_t>(GetSerializeSize(blockundo))};
if (!FindUndoPos(state, block.nFile, pos, blockundo_size + UNDO_DATA_DISK_OVERHEAD)) {
LogError("%s: FindUndoPos failed\n", __func__);
return false;
}
if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash())) {
AutoFile fileout{m_undo_file_seq.Open(pos, false), 0}; // We'll obfuscate ourselves
if (fileout.IsNull()) {
LogError("%s: OpenUndoFile failed\n", __func__);
return FatalError(m_opts.notifications, state, _("Failed to write undo data."));
}
{
DataStream header;
header.reserve(BLOCK_SERIALIZATION_HEADER_SIZE);
header << GetParams().MessageStart() << blockundo_size;
m_obfuscation(header, pos.nPos);
fileout.write(header);
}
pos.nPos += BLOCK_SERIALIZATION_HEADER_SIZE;
{
HashWriter hasher;
hasher << block.pprev->GetBlockHash();
hasher << blockundo;
DataStream undo_data;
undo_data.reserve(blockundo_size + sizeof(uint256));
undo_data << blockundo << hasher.GetHash();
m_obfuscation(undo_data, pos.nPos);
fileout.write(undo_data);
}
// rev files are written in block height order, whereas blk files are written as blocks come in (often out of order)
// we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height
// in the block file info as below; note that this does not catch the case where the undo writes are keeping up
// with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in
// the FindNextBlockPos function
if (_pos.nFile < cursor.file_num && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) {
if (pos.nFile < cursor.file_num && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[pos.nFile].nHeightLast) {
// Do not propagate the return code, a failed flush here should not
// be an indication for a failed write. If it were propagated here,
// the caller would assume the undo data not to be written, when in
// fact it is. Note though, that a failed flush might leave the data
// file untrimmed.
if (!FlushUndoFile(_pos.nFile, true)) {
LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, "Failed to flush undo file %05i\n", _pos.nFile);
if (!FlushUndoFile(pos.nFile, true)) {
LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, "Failed to flush undo file %05i\n", pos.nFile);
}
} else if (_pos.nFile == cursor.file_num && block.nHeight > cursor.undo_height) {
} else if (pos.nFile == cursor.file_num && block.nHeight > cursor.undo_height) {
cursor.undo_height = block.nHeight;
}
// update nUndoPos in block index
block.nUndoPos = _pos.nPos;
block.nUndoPos = pos.nPos;
block.nStatus |= BLOCK_HAVE_UNDO;
m_dirty_blockindex.insert(&block);
}
@ -1026,20 +1010,26 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid
return true;
}
bool BlockManager::ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const
bool BlockManager::ReadBlockFromDisk(CBlock& block, FlatFilePos pos) const
{
block.SetNull();
// Open history file to read
if (pos.nPos < BLOCK_SERIALIZATION_HEADER_SIZE) return false;
uint32_t blk_size;
pos.nPos -= sizeof blk_size;
AutoFile filein{OpenBlockFile(pos, true)};
if (filein.IsNull()) {
LogError("%s: OpenBlockFile failed for %s\n", __func__, pos.ToString());
return false;
}
// Read block
try {
filein >> TX_WITH_WITNESS(block);
filein >> blk_size;
if (blk_size > MAX_SIZE) throw std::runtime_error{strprintf("Refusing to read block of size: %d", blk_size)};
std::vector<uint8_t> mem(blk_size);
filein >> Span{mem};
SpanReader(mem) >> TX_WITH_WITNESS(block);
} catch (const std::exception& e) {
LogError("%s: Deserialize or I/O error - %s at %s\n", __func__, e.what(), pos.ToString());
return false;
@ -1119,25 +1109,41 @@ bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatF
return true;
}
FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight)
FlatFilePos BlockManager::SaveBlock(const CBlock& block, int nHeight)
{
unsigned int nBlockSize = ::GetSerializeSize(TX_WITH_WITNESS(block));
// Account for the 4 magic message start bytes + the 4 length bytes (8 bytes total,
// defined as BLOCK_SERIALIZATION_HEADER_SIZE)
nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE);
FlatFilePos blockPos{FindNextBlockPos(nBlockSize, nHeight, block.GetBlockTime())};
if (blockPos.IsNull()) {
const uint32_t block_size{static_cast<uint32_t>(GetSerializeSize(TX_WITH_WITNESS(block)))};
FlatFilePos pos{FindNextBlockPos(BLOCK_SERIALIZATION_HEADER_SIZE + block_size, nHeight, block.GetBlockTime())};
if (pos.IsNull()) {
LogError("%s: FindNextBlockPos failed\n", __func__);
return FlatFilePos();
}
if (!WriteBlockToDisk(block, blockPos)) {
AutoFile fileout{m_block_file_seq.Open(pos, false), 0}; // We'll obfuscate ourselves
if (fileout.IsNull()) {
LogError("%s: OpenBlockFile failed\n", __func__);
m_opts.notifications.fatalError(_("Failed to write block."));
return FlatFilePos();
}
return blockPos;
{
DataStream header;
header.reserve(BLOCK_SERIALIZATION_HEADER_SIZE);
header << GetParams().MessageStart() << block_size;
m_obfuscation(header, pos.nPos);
fileout.write(header);
}
pos.nPos += BLOCK_SERIALIZATION_HEADER_SIZE;
{
DataStream block_data;
block_data.reserve(block_size);
block_data << TX_WITH_WITNESS(block);
m_obfuscation(block_data, pos.nPos);
fileout.write(block_data);
}
return pos;
}
static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
static Obfuscation InitBlocksdirXorKey(const BlockManager::Options& opts)
{
// Bytes are serialized without length indicator, so this is also the exact
// size of the XOR-key file.
@ -1174,12 +1180,12 @@ static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
};
}
LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(xor_key));
return std::vector<std::byte>{xor_key.begin(), xor_key.end()};
return Obfuscation{xor_key};
}
BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts)
: m_prune_mode{opts.prune_target > 0},
m_xor_key{InitBlocksdirXorKey(opts)},
m_obfuscation{InitBlocksdirXorKey(opts)},
m_opts{std::move(opts)},
m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}},
m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}},

29
src/node/blockstorage.h
View file

@ -74,8 +74,11 @@ static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB
/** The maximum size of a blk?????.dat file (since 0.8) */
static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
/** Size of header written by WriteBlockToDisk before a serialized CBlock */
static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = std::tuple_size_v<MessageStartChars> + sizeof(unsigned int);
/** Size of header written by SaveBlock before a serialized CBlock (8 bytes) */
static constexpr uint32_t BLOCK_SERIALIZATION_HEADER_SIZE = std::tuple_size_v<MessageStartChars> + sizeof(uint32_t);
/** Total overhead when writing undo data: header (8 bytes) plus checksum (32 bytes) */
static constexpr uint32_t UNDO_DATA_DISK_OVERHEAD = BLOCK_SERIALIZATION_HEADER_SIZE + uint256::size();
// Because validation code takes pointers to the map's CBlockIndex objects, if
// we ever switch to another associative container, we need to either use a
@ -161,7 +164,7 @@ private:
* blockfile info, and checks if there is enough disk space to save the block.
*
* The nAddSize argument passed to this function should include not just the size of the serialized CBlock, but also the size of
* separator fields which are written before it by WriteBlockToDisk (BLOCK_SERIALIZATION_HEADER_SIZE).
* separator fields (BLOCK_SERIALIZATION_HEADER_SIZE).
*/
[[nodiscard]] FlatFilePos FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime);
[[nodiscard]] bool FlushChainstateBlockFile(int tip_height);
@ -169,15 +172,6 @@ private:
AutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const;
/**
* Write a block to disk. The pos argument passed to this function is modified by this call. Before this call, it should
* point to an unused file location where separator fields will be written, followed by the serialized CBlock data.
* After this call, it will point to the beginning of the serialized CBlock data, after the separator fields
* (BLOCK_SERIALIZATION_HEADER_SIZE)
*/
bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const;
bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const;
/* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */
void FindFilesToPruneManual(
std::set<int>& setFilesToPrune,
@ -241,7 +235,7 @@ private:
const bool m_prune_mode;
const std::vector<std::byte> m_xor_key;
const Obfuscation m_obfuscation;
/** Dirty block index entries. */
std::set<CBlockIndex*> m_dirty_blockindex;
@ -330,7 +324,7 @@ public:
/** Get block file info entry for one block file */
CBlockFileInfo* GetBlockFileInfo(size_t n);
bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
bool SaveBlockUndo(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block)
EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
/** Store block on disk and update block file statistics.
@ -341,14 +335,13 @@ public:
* @returns in case of success, the position to which the block was written to
* in case of an error, an empty FlatFilePos
*/
FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight);
FlatFilePos SaveBlock(const CBlock& block, int nHeight);
/** Update blockfile info while processing a block during reindex. The block must be available on disk.
*
* @param[in] block the block being processed
* @param[in] nHeight the height of the block
* @param[in] pos the position of the serialized CBlock on disk. This is the position returned
* by WriteBlockToDisk pointing at the CBlock, not the separator fields before it
* @param[in] pos the position of the serialized CBlock on disk
*/
void UpdateBlockInfo(const CBlock& block, unsigned int nHeight, const FlatFilePos& pos);
@ -421,7 +414,7 @@ public:
void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const;
/** Functions for disk access for blocks */
bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const;
bool ReadBlockFromDisk(CBlock& block, FlatFilePos pos) const;
bool ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const;
bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos) const;

17
src/node/mempool_persist.cpp
View file

@ -58,15 +58,15 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active
try {
uint64_t version;
file >> version;
std::vector<std::byte> xor_key;
if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) {
// Leave XOR-key empty
file.SetObfuscation(0);
} else if (version == MEMPOOL_DUMP_VERSION) {
file >> xor_key;
Obfuscation obfuscation{0};
file >> obfuscation;
file.SetObfuscation(obfuscation);
} else {
return false;
}
file.SetXor(xor_key);
uint64_t total_txns_to_load;
file >> total_txns_to_load;
uint64_t txns_tried = 0;
@ -177,12 +177,13 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock
const uint64_t version{pool.m_opts.persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION};
file << version;
std::vector<std::byte> xor_key(8);
if (!pool.m_opts.persist_v1_dat) {
FastRandomContext{}.fillrand(xor_key);
file << xor_key;
const Obfuscation obfuscation{FastRandomContext{}.rand64()};
file << obfuscation;
file.SetObfuscation(obfuscation);
} else {
file.SetObfuscation(0);
}
file.SetXor(xor_key);
uint64_t mempool_transactions_to_write(vinfo.size());
file << mempool_transactions_to_write;

86
src/obfuscation.h Normal file
View file

@ -0,0 +1,86 @@
// Copyright (c) 2009-present The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef BITCOIN_OBFUSCATION_H
#define BITCOIN_OBFUSCATION_H
#include <array>
#include <cassert>
#include <cstdint>
#include <random>
#include <span.h>
#include <util/check.h>
#include <cstring>
#include <climits>
#include <serialize.h>
class Obfuscation
{
public:
static constexpr size_t SIZE_BYTES{sizeof(uint64_t)};
private:
std::array<uint64_t, SIZE_BYTES> rotations; // Cached key rotations
void SetRotations(const uint64_t key)
{
for (size_t i{0}; i < SIZE_BYTES; ++i)
{
size_t key_rotation_bits{CHAR_BIT * i};
if constexpr (std::endian::native == std::endian::big) key_rotation_bits *= -1;
rotations[i] = std::rotr(key, key_rotation_bits);
}
}
static uint64_t ToUint64(const Span<const std::byte> key_span)
{
uint64_t key{};
std::memcpy(&key, key_span.data(), SIZE_BYTES);
return key;
}
static void Xor(Span<std::byte> write, const uint64_t key, const size_t size)
{
assert(size <= write.size());
uint64_t raw{};
std::memcpy(&raw, write.data(), size);
raw ^= key;
std::memcpy(write.data(), &raw, size);
}
public:
Obfuscation(const uint64_t key) { SetRotations(key); }
Obfuscation(const Span<const std::byte> key_span) : Obfuscation(ToUint64(key_span)) {}
Obfuscation(const std::array<const std::byte, SIZE_BYTES>& key_arr) : Obfuscation(ToUint64(key_arr)) {}
Obfuscation(const std::vector<uint8_t>& key_vec) : Obfuscation(MakeByteSpan(key_vec)) {}
uint64_t Key() const { return rotations[0]; }
operator bool() const { return Key() != 0; }
void operator()(Span<std::byte> write, const size_t key_offset_bytes = 0) const
{
if (!*this) return;
const uint64_t rot_key{rotations[key_offset_bytes % SIZE_BYTES]}; // Continue obfuscation from where we left off
for (; write.size() >= SIZE_BYTES; write = write.subspan(SIZE_BYTES)) { // Process multiple bytes at a time
Xor(write, rot_key, SIZE_BYTES);
}
Xor(write, rot_key, write.size());
}
template <typename Stream>
void Serialize(Stream& s) const
{
std::vector<std::byte> bytes(SIZE_BYTES);
std::memcpy(bytes.data(), &rotations[0], SIZE_BYTES);
s << bytes;
}
template <typename Stream>
void Unserialize(Stream& s)
{
std::vector<std::byte> bytes(SIZE_BYTES);
s >> bytes;
SetRotations(ToUint64(bytes));
}
};
#endif // BITCOIN_OBFUSCATION_H

19
src/streams.cpp
View file

@ -9,8 +9,7 @@
#include <array>
AutoFile::AutoFile(std::FILE* file, std::vector<std::byte> data_xor)
: m_file{file}, m_xor{std::move(data_xor)}
AutoFile::AutoFile(std::FILE* file, const Obfuscation& obfuscation) : m_file{file}, m_obfuscation{obfuscation}
{
if (!IsNull()) {
auto pos{std::ftell(m_file)};
@ -21,12 +20,12 @@ AutoFile::AutoFile(std::FILE* file, std::vector<std::byte> data_xor)
std::size_t AutoFile::detail_fread(Span<std::byte> dst)
{
if (!m_file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr");
size_t ret = std::fread(dst.data(), 1, dst.size(), m_file);
if (!m_xor.empty()) {
if (!m_position.has_value()) throw std::ios_base::failure("AutoFile::read: position unknown");
util::Xor(dst.subspan(0, ret), m_xor, *m_position);
const size_t ret = std::fread(dst.data(), 1, dst.size(), m_file);
if (m_obfuscation) {
if (!m_position) throw std::ios_base::failure("AutoFile::read: position unknown");
m_obfuscation(dst, *m_position);
}
if (m_position.has_value()) *m_position += ret;
if (m_position) *m_position += ret;
return ret;
}
@ -81,7 +80,7 @@ void AutoFile::ignore(size_t nSize)
void AutoFile::write(Span<const std::byte> src)
{
if (!m_file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr");
if (m_xor.empty()) {
if (!m_obfuscation) {
if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) {
throw std::ios_base::failure("AutoFile::write: write failed");
}
@ -91,8 +90,8 @@ void AutoFile::write(Span<const std::byte> src)
std::array<std::byte, 4096> buf;
while (src.size() > 0) {
auto buf_now{Span{buf}.first(std::min<size_t>(src.size(), buf.size()))};
std::copy(src.begin(), src.begin() + buf_now.size(), buf_now.begin());
util::Xor(buf_now, m_xor, *m_position);
std::copy_n(src.begin(), buf_now.size(), buf_now.begin());
m_obfuscation(buf_now, *m_position);
if (std::fwrite(buf_now.data(), 1, buf_now.size(), m_file) != buf_now.size()) {
throw std::ios_base::failure{"XorFile::write: failed"};
}

40
src/streams.h
View file

@ -6,6 +6,7 @@
#ifndef BITCOIN_STREAMS_H
#define BITCOIN_STREAMS_H
#include <obfuscation.h>
#include <serialize.h>
#include <span.h>
#include <support/allocators/zeroafterfree.h>
@ -21,30 +22,8 @@
#include <stdint.h>
#include <string.h>
#include <string>
#include <utility>
#include <vector>
namespace util {
inline void Xor(Span<std::byte> write, Span<const std::byte> key, size_t key_offset = 0)
{
if (key.size() == 0) {
return;
}
key_offset %= key.size();
for (size_t i = 0, j = key_offset; i != write.size(); i++) {
write[i] ^= key[j++];
// This potentially acts on very many bytes of data, so it's
// important that we calculate `j`, i.e. the `key` index in this
// way instead of doing a %, which would effectively be a division
// for each byte Xor'd -- much slower than need be.
if (j == key.size())
j = 0;
}
}
} // namespace util
/* Minimal stream for overwriting and/or appending to an existing byte vector
*
* The referenced vector will grow as necessary
@ -261,21 +240,16 @@ public:
return (*this);
}
template<typename T>
template <typename T>
DataStream& operator>>(T&& obj)
{
::Unserialize(*this, obj);
return (*this);
}
/**
* XOR the contents of this stream with a certain key.
*
* @param[in] key The key used to XOR the data in this stream.
*/
void Xor(const std::vector<unsigned char>& key)
void Obfuscate(const Obfuscation& obfuscation)
{
util::Xor(MakeWritableByteSpan(*this), MakeByteSpan(key));
if (obfuscation) obfuscation(MakeWritableByteSpan(*this));
}
/** Compute total memory usage of this object (own memory + any dynamic memory). */
@ -392,11 +366,11 @@ class AutoFile
{
protected:
std::FILE* m_file;
std::vector<std::byte> m_xor;
Obfuscation m_obfuscation;
std::optional<int64_t> m_position;
public:
explicit AutoFile(std::FILE* file, std::vector<std::byte> data_xor={});
explicit AutoFile(std::FILE* file, const Obfuscation& obfuscation = 0);
~AutoFile() { fclose(); }
@ -428,7 +402,7 @@ public:
bool IsNull() const { return m_file == nullptr; }
/** Continue with a different XOR key */
void SetXor(std::vector<std::byte> data_xor) { m_xor = data_xor; }
void SetObfuscation(const Obfuscation& obfuscation) { m_obfuscation = obfuscation; }
/** Implementation detail, only used internally. */
std::size_t detail_fread(Span<std::byte> dst);

8
src/test/blockmanager_tests.cpp
View file

@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos)
};
BlockManager blockman{*Assert(m_node.shutdown_signal), blockman_opts};
// simulate adding a genesis block normally
BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0).nPos, BLOCK_SERIALIZATION_HEADER_SIZE);
BOOST_CHECK_EQUAL(blockman.SaveBlock(params->GenesisBlock(), 0).nPos, BLOCK_SERIALIZATION_HEADER_SIZE);
// simulate what happens during reindex
// simulate a well-formed genesis block being found at offset 8 in the blk00000.dat file
// the block is found at offset 8 because there is an 8 byte serialization header
@ -49,7 +49,7 @@ BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos)
// this is a check to make sure that https://github.com/bitcoin/bitcoin/issues/21379 does not recur
// 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293
// add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301
FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1)};
FlatFilePos actual{blockman.SaveBlock(params->GenesisBlock(), 1)};
BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(TX_WITH_WITNESS(params->GenesisBlock())) + BLOCK_SERIALIZATION_HEADER_SIZE);
}
@ -158,10 +158,10 @@ BOOST_AUTO_TEST_CASE(blockmanager_flush_block_file)
BOOST_CHECK_EQUAL(blockman.CalculateCurrentUsage(), 0);
// Write the first block to a new location.
FlatFilePos pos1{blockman.SaveBlockToDisk(block1, /*nHeight=*/1)};
FlatFilePos pos1{blockman.SaveBlock(block1, /*nHeight=*/1)};
// Write second block
FlatFilePos pos2{blockman.SaveBlockToDisk(block2, /*nHeight=*/2)};
FlatFilePos pos2{blockman.SaveBlock(block2, /*nHeight=*/2)};
// Two blocks in the file
BOOST_CHECK_EQUAL(blockman.CalculateCurrentUsage(), (TEST_BLOCK_SIZE + BLOCK_SERIALIZATION_HEADER_SIZE) * 2);

18
src/test/dbwrapper_tests.cpp
View file

@ -14,16 +14,6 @@
using util::ToString;
// Test if a string consists entirely of null characters
static bool is_null_key(const std::vector<unsigned char>& key) {
bool isnull = true;
for (unsigned int i = 0; i < key.size(); i++)
isnull &= (key[i] == '\x00');
return isnull;
}
BOOST_FIXTURE_TEST_SUITE(dbwrapper_tests, BasicTestingSetup)
BOOST_AUTO_TEST_CASE(dbwrapper)
@ -37,7 +27,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper)
uint256 res;
// Ensure that we're doing real obfuscation when obfuscate=true
BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscateKey(dbw)));
BOOST_CHECK(obfuscate == dbwrapper_private::GetObfuscation(dbw));
BOOST_CHECK(dbw.Write(key, in));
BOOST_CHECK(dbw.Read(key, res));
@ -57,7 +47,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_basic_data)
bool res_bool;
// Ensure that we're doing real obfuscation when obfuscate=true
BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscateKey(dbw)));
BOOST_CHECK(obfuscate == dbwrapper_private::GetObfuscation(dbw));
//Simulate block raw data - "b + block hash"
std::string key_block = "b" + m_rng.rand256().ToString();
@ -232,7 +222,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate)
BOOST_CHECK_EQUAL(res2.ToString(), in.ToString());
BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data
BOOST_CHECK(is_null_key(dbwrapper_private::GetObfuscateKey(odbw))); // The key should be an empty string
BOOST_CHECK(!dbwrapper_private::GetObfuscation(odbw));
uint256 in2 = m_rng.rand256();
uint256 res3;
@ -269,7 +259,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex)
// Check that the key/val we wrote with unobfuscated wrapper doesn't exist
uint256 res2;
BOOST_CHECK(!odbw.Read(key, res2));
BOOST_CHECK(!is_null_key(dbwrapper_private::GetObfuscateKey(odbw)));
BOOST_CHECK(dbwrapper_private::GetObfuscation(odbw));
uint256 in2 = m_rng.rand256();
uint256 res3;

2
src/test/fuzz/autofile.cpp
View file

@ -20,7 +20,7 @@ FUZZ_TARGET(autofile)
FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider};
AutoFile auto_file{
fuzzed_file_provider.open(),
ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider),
fuzzed_data_provider.ConsumeIntegral<uint64_t>()
};
LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100)
{

2
src/test/fuzz/buffered_file.cpp
View file

@ -22,7 +22,7 @@ FUZZ_TARGET(buffered_file)
std::optional<BufferedFile> opt_buffered_file;
AutoFile fuzzed_file{
fuzzed_file_provider.open(),
ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider),
fuzzed_data_provider.ConsumeIntegral<uint64_t>()
};
try {
auto n_buf_size = fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096);

134
src/test/streams_tests.cpp
View file

@ -14,16 +14,121 @@ using namespace std::string_literals;
BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup)
BOOST_AUTO_TEST_CASE(obfuscation_constructors)
{
constexpr uint64_t test_key = 0x0123456789ABCDEF;
// Direct uint64_t constructor
const Obfuscation obf1{test_key};
BOOST_CHECK_EQUAL(obf1.Key(), test_key);
// Span constructor
std::array<std::byte, Obfuscation::SIZE_BYTES> key_bytes{};
std::memcpy(key_bytes.data(), &test_key, Obfuscation::SIZE_BYTES);
const Obfuscation obf2{Span{key_bytes}};
BOOST_CHECK_EQUAL(obf2.Key(), test_key);
// std::array<std:byte> constructor
const Obfuscation obf3{key_bytes};
BOOST_CHECK_EQUAL(obf3.Key(), test_key);
// std::vector<uint8_t> constructor
std::vector<uint8_t> uchar_key(Obfuscation::SIZE_BYTES);
std::memcpy(uchar_key.data(), &test_key, uchar_key.size());
const Obfuscation obf4{uchar_key};
BOOST_CHECK_EQUAL(obf4.Key(), test_key);
}
BOOST_AUTO_TEST_CASE(obfuscation_serialize)
{
const Obfuscation original{0xDEADBEEF};
// Serialize
DataStream ds;
ds << original;
BOOST_CHECK_EQUAL(ds.size(), 1 + Obfuscation::SIZE_BYTES); // serialized as a vector
// Deserialize
Obfuscation recovered{0};
ds >> recovered;
BOOST_CHECK_EQUAL(recovered.Key(), original.Key());
}
BOOST_AUTO_TEST_CASE(obfuscation_empty)
{
const Obfuscation null_obf{0};
BOOST_CHECK(!null_obf);
}
BOOST_AUTO_TEST_CASE(xor_bytes_reference)
{
auto expected_xor{[](std::span<std::byte> write, const std::span<const std::byte> key, size_t key_offset) {
for (auto& b : write) {
b ^= key[key_offset++ % key.size()];
}
}};
FastRandomContext rng{/*fDeterministic=*/false};
for (size_t test{0}; test < 100; ++test) {
const size_t write_size{1 + rng.randrange(100U)};
const size_t key_offset{rng.randrange(3 * 8U)}; // Should wrap around
const auto key_bytes{rng.randbytes<std::byte>(Obfuscation::SIZE_BYTES)};
const Obfuscation obfuscation{key_bytes};
std::vector expected{rng.randbytes<std::byte>(write_size)};
std::vector actual{expected};
expected_xor(expected, key_bytes, key_offset);
obfuscation(actual, key_offset);
BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(), actual.begin(), actual.end());
}
}
BOOST_AUTO_TEST_CASE(xor_roundtrip_random_chunks)
{
auto apply_random_xor_chunks{[](std::span<std::byte> write, const Obfuscation& obfuscation, FastRandomContext& rng) {
for (size_t offset{0}; offset < write.size();) {
const size_t chunk_size{1 + rng.randrange(write.size() - offset)};
obfuscation(write.subspan(offset, chunk_size), offset);
offset += chunk_size;
}
}};
FastRandomContext rng{/*fDeterministic=*/false};
for (size_t test{0}; test < 100; ++test) {
const size_t write_size{1 + rng.randrange(100U)};
const std::vector original{rng.randbytes<std::byte>(write_size)};
std::vector roundtrip{original};
const auto key_bytes{rng.randbytes<std::byte>(Obfuscation::SIZE_BYTES)};
const Obfuscation obfuscation{key_bytes};
apply_random_xor_chunks(roundtrip, obfuscation, rng);
const bool all_zero = !obfuscation || (HexStr(key_bytes).find_first_not_of('0') >= write_size * 2);
BOOST_CHECK_EQUAL(original != roundtrip, !all_zero);
apply_random_xor_chunks(roundtrip, obfuscation, rng);
BOOST_CHECK(original == roundtrip);
}
}
BOOST_AUTO_TEST_CASE(xor_file)
{
fs::path xor_path{m_args.GetDataDirBase() / "test_xor.bin"};
auto raw_file{[&](const auto& mode) { return fsbridge::fopen(xor_path, mode); }};
const std::vector<uint8_t> test1{1, 2, 3};
const std::vector<uint8_t> test2{4, 5};
const std::vector<std::byte> xor_pat{std::byte{0xff}, std::byte{0x00}};
constexpr std::array xor_pat{std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}};
uint64_t xor_key;
std::memcpy(&xor_key, xor_pat.data(), sizeof xor_key);
{
// Check errors for missing file
AutoFile xor_file{raw_file("rb"), xor_pat};
AutoFile xor_file{raw_file("rb"), xor_key};
BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullpt"});
BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullpt"});
BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullpt"});
@ -35,7 +140,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
#else
const char* mode = "wbx";
#endif
AutoFile xor_file{raw_file(mode), xor_pat};
AutoFile xor_file{raw_file(mode), xor_key};
xor_file << test1 << test2;
}
{
@ -48,7 +153,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
BOOST_CHECK_EXCEPTION(non_xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"});
}
{
AutoFile xor_file{raw_file("rb"), xor_pat};
AutoFile xor_file{raw_file("rb"), xor_key};
std::vector<std::byte> read1, read2;
xor_file >> read1 >> read2;
BOOST_CHECK_EQUAL(HexStr(read1), HexStr(test1));
@ -57,7 +162,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"});
}
{
AutoFile xor_file{raw_file("rb"), xor_pat};
AutoFile xor_file{raw_file("rb"), xor_key};
std::vector<std::byte> read2;
// Check that ignore works
xor_file.ignore(4);
@ -73,7 +178,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_writer)
{
unsigned char a(1);
unsigned char b(2);
unsigned char bytes[] = { 3, 4, 5, 6 };
unsigned char bytes[] = {3, 4, 5, 6};
std::vector<unsigned char> vch;
// Each test runs twice. Serializing a second time at the same starting
@ -225,29 +330,30 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor)
// Degenerate case
{
DataStream ds{in};
ds.Xor({0x00, 0x00});
Obfuscation{0}(ds);
BOOST_CHECK_EQUAL(""s, ds.str());
}
in.push_back(std::byte{0x0f});
in.push_back(std::byte{0xf0});
// Single character key
{
const Obfuscation obfuscation{{std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}}};
DataStream ds{in};
ds.Xor({0xff});
obfuscation(ds);
BOOST_CHECK_EQUAL("\xf0\x0f"s, ds.str());
}
// Multi character key
in.clear();
in.push_back(std::byte{0xf0});
in.push_back(std::byte{0x0f});
{
const Obfuscation obfuscation{{std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}}};
DataStream ds{in};
ds.Xor({0xff, 0x0f});
obfuscation(ds);
BOOST_CHECK_EQUAL("\x0f\x00"s, ds.str());
}
}
@ -270,7 +376,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
BOOST_CHECK(false);
} catch (const std::exception& e) {
BOOST_CHECK(strstr(e.what(),
"Rewind limit must be less than buffer size") != nullptr);
"Rewind limit must be less than buffer size") != nullptr);
}
// The buffer is 25 bytes, allow rewinding 10 bytes.
@ -359,7 +465,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file)
BOOST_CHECK(false);
} catch (const std::exception& e) {
BOOST_CHECK(strstr(e.what(),
"BufferedFile::Fill: end of file") != nullptr);
"BufferedFile::Fill: end of file") != nullptr);
}
// Attempting to read beyond the end sets the EOF indicator.
BOOST_CHECK(bf.eof());

6
src/validation.cpp
View file

@ -2747,7 +2747,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state,
return true;
}
if (!m_blockman.WriteUndoDataForBlock(blockundo, state, *pindex)) {
if (!m_blockman.SaveBlockUndo(blockundo, state, *pindex)) {
return false;
}
@ -4564,7 +4564,7 @@ bool ChainstateManager::AcceptBlock(const std::shared_ptr<const CBlock>& pblock,
blockPos = *dbp;
m_blockman.UpdateBlockInfo(block, pindex->nHeight, blockPos);
} else {
blockPos = m_blockman.SaveBlockToDisk(block, pindex->nHeight);
blockPos = m_blockman.SaveBlock(block, pindex->nHeight);
if (blockPos.IsNull()) {
state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__));
return false;
@ -5062,7 +5062,7 @@ bool Chainstate::LoadGenesisBlock()
try {
const CBlock& block = params.GenesisBlock();
FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0)};
FlatFilePos blockPos{m_blockman.SaveBlock(block, 0)};
if (blockPos.IsNull()) {
LogError("%s: writing genesis block to disk failed\n", __func__);
return false;