optimization: Xor 64 bits together instead of byte-by-byte

`util::Xor` method was split out into more focused parts:
* one which assumes tha the `uint64_t` key is properly aligned, doing the first few xors as 64 bits (the memcpy is eliminated in most compilers), and the last iteration is optimized for 8/16/32 bytes.
* an unaligned `uint64_t` key with a `key_offset` parameter which is rotated to accommodate the data (adjusting for endianness).
* a legacy `std::vector<std::byte>` key with an asserted 8 byte size, converted to `uint64_t`.

Note that the default statement alone would pass the tests, but would be very slow, since the 1, 2 and 4 byte versions won't be specialized by the compiler, hence the switch.

Asserts were added throughout the code to make sure every such vector has length 8, since in the next commit we're converting all of them to `uint64_t`.

refactor: Migrate fixed-size obfuscation end-to-end from `std::vector<std::byte>` to `uint64_t`

Since `util::Xor` accepts `uint64_t` values, we're eliminating any repeated vector-to-uint64_t conversions going back to the loading/saving of these values (we're still serializing them as vectors, but converting as soon as possible to `uint64_t`). This is the reason the tests still generate vector values and convert to `uint64_t` later instead of generating it directly.

We're also short-circuit `Xor` calls with 0 key values early to avoid unnecessary calculations (e.g. `MakeWritableByteSpan`) - even assuming that XOR is never called for 0.

>  cmake -B build -DBUILD_BENCH=ON -DCMAKE_BUILD_TYPE=Release \
&& cmake --build build -j$(nproc) \
&& build/src/bench/bench_bitcoin -filter='XorHistogram|AutoFileXor' -min-time=10000

C++ compiler .......................... AppleClang 16.0.0.16000026

|             ns/byte |              byte/s |    err% |     total | benchmark
|--------------------:|--------------------:|--------:|----------:|:----------
|                0.09 |   10,799,585,470.46 |    1.3% |     11.00 | `AutoFileXor`
|                0.14 |    7,144,743,097.97 |    0.2% |     11.01 | `XorHistogram`

C++ compiler .......................... GNU 13.2.0

|             ns/byte |              byte/s |    err% |        ins/byte |        cyc/byte |    IPC |       bra/byte |   miss% |     total | benchmark
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
|                0.59 |    1,706,433,032.76 |    0.1% |            0.00 |            0.00 |  0.620 |           0.00 |    1.8% |     11.01 | `AutoFileXor`
|                0.47 |    2,145,375,849.71 |    0.0% |            0.95 |            1.48 |  0.642 |           0.20 |    9.6% |     10.93 | `XorHistogram`

----

A few other benchmarks that seem to have improved as well (tested with Clang only):
Before:

|               ns/op |                op/s |    err% |     total | benchmark
|--------------------:|--------------------:|--------:|----------:|:----------
|        2,237,168.64 |              446.99 |    0.3% |     10.91 | `ReadBlockFromDiskTest`
|          748,837.59 |            1,335.40 |    0.2% |     10.68 | `ReadRawBlockFromDiskTest`

After:

|               ns/op |                op/s |    err% |     total | benchmark
|--------------------:|--------------------:|--------:|----------:|:----------
|        1,827,436.12 |              547.21 |    0.7% |     10.95 | `ReadBlockFromDiskTest`
|           49,276.48 |           20,293.66 |    0.2% |     10.99 | `ReadRawBlockFromDiskTest`
This commit is contained in:
Lőrinc 2024-12-21 16:15:04 +01:00
parent df516ee5e3
commit 898a07e2ab
13 changed files with 238 additions and 169 deletions

View file

@ -17,13 +17,12 @@ static void Xor(benchmark::Bench& bench)
FastRandomContext rng{/*fDeterministic=*/true};
auto test_data{rng.randbytes<std::byte>(1 << 20)};
std::vector key_bytes{rng.randbytes<std::byte>(8)};
uint64_t key;
std::memcpy(&key, key_bytes.data(), 8);
const Obfuscation obfuscation{rng.rand64()};
assert(obfuscation);
size_t offset{0};
bench.batch(test_data.size()).unit("byte").run([&] {
util::Xor(test_data, key_bytes, offset++);
obfuscation(test_data, offset++);
ankerl::nanobench::doNotOptimizeAway(test_data);
});
}

View file

@ -171,7 +171,7 @@ void CDBBatch::Clear()
void CDBBatch::WriteImpl(Span<const std::byte> key, DataStream& ssValue)
{
leveldb::Slice slKey(CharCast(key.data()), key.size());
ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
dbwrapper_private::GetObfuscation(parent)(ssValue);
leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size());
m_impl_batch->batch.Put(slKey, slValue);
// LevelDB serializes writes as:
@ -220,7 +220,11 @@ struct LevelDBContext {
};
CDBWrapper::CDBWrapper(const DBParams& params)
: m_db_context{std::make_unique<LevelDBContext>()}, m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only}
: m_db_context{std::make_unique<LevelDBContext>()},
m_name{fs::PathToString(params.path.stem())},
m_obfuscation{0},
m_path{params.path},
m_is_memory{params.memory_only}
{
DBContext().penv = nullptr;
DBContext().readoptions.verify_checksums = true;
@ -255,24 +259,23 @@ CDBWrapper::CDBWrapper(const DBParams& params)
LogPrintf("Finished database compaction of %s\n", fs::PathToString(params.path));
}
// The base-case obfuscation key, which is a noop.
obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
if (!key_exists && params.obfuscate && IsEmpty()) {
// Initialize non-degenerate obfuscation if it won't upset
// existing, non-obfuscated data.
std::vector<unsigned char> new_key = CreateObfuscateKey();
m_obfuscation = 0; // Needed for unobfuscated Read
std::vector<unsigned char> obfuscate_key_vector(Obfuscation::SIZE_BYTES, '\000');
const bool key_missing = !Read(OBFUSCATE_KEY_KEY, obfuscate_key_vector);
if (key_missing && params.obfuscate && IsEmpty()) {
// Initialize non-degenerate obfuscation if it won't upset existing, non-obfuscated data.
std::vector<uint8_t> new_key(Obfuscation::SIZE_BYTES);
GetRandBytes(new_key);
// Write `new_key` so we don't obfuscate the key with itself
Write(OBFUSCATE_KEY_KEY, new_key);
obfuscate_key = new_key;
obfuscate_key_vector = new_key;
LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key));
LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key_vector));
}
LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key));
LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key_vector));
m_obfuscation = obfuscate_key_vector;
obfuscate_key_vector.clear();
}
CDBWrapper::~CDBWrapper()
@ -323,19 +326,6 @@ size_t CDBWrapper::DynamicMemoryUsage() const
// past the null-terminator.
const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
/**
* Returns a string (consisting of 8 random bytes) suitable for use as an
* obfuscating XOR key.
*/
std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
{
std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES);
GetRandBytes(ret);
return ret;
}
std::optional<std::string> CDBWrapper::ReadImpl(Span<const std::byte> key) const
{
leveldb::Slice slKey(CharCast(key.data()), key.size());
@ -418,10 +408,5 @@ void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); }
void CDBIterator::Next() { m_impl_iter->iter->Next(); }
namespace dbwrapper_private {
const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w)
{
return w.obfuscate_key;
}
Obfuscation GetObfuscation(const CDBWrapper& w) { return w.m_obfuscation; }
} // namespace dbwrapper_private

View file

@ -63,8 +63,7 @@ namespace dbwrapper_private {
* Database obfuscation should be considered an implementation detail of the
* specific database.
*/
const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w);
Obfuscation GetObfuscation(const CDBWrapper&);
}; // namespace dbwrapper_private
bool DestroyDB(const std::string& path_str);
@ -168,7 +167,7 @@ public:
template<typename V> bool GetValue(V& value) {
try {
DataStream ssValue{GetValueImpl()};
ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
dbwrapper_private::GetObfuscation(parent)(ssValue);
ssValue >> value;
} catch (const std::exception&) {
return false;
@ -181,7 +180,7 @@ struct LevelDBContext;
class CDBWrapper
{
friend const std::vector<unsigned char>& dbwrapper_private::GetObfuscateKey(const CDBWrapper &w);
friend Obfuscation dbwrapper_private::GetObfuscation(const CDBWrapper&);
private:
//! holds all leveldb-specific fields of this class
std::unique_ptr<LevelDBContext> m_db_context;
@ -190,16 +189,11 @@ private:
std::string m_name;
//! a key used for optional XOR-obfuscation of the database
std::vector<unsigned char> obfuscate_key;
Obfuscation m_obfuscation;
//! the key under which the obfuscation key is stored
static const std::string OBFUSCATE_KEY_KEY;
//! the length of the obfuscate key in number of bytes
static const unsigned int OBFUSCATE_KEY_NUM_BYTES;
std::vector<unsigned char> CreateObfuscateKey() const;
//! path to filesystem storage
const fs::path m_path;
@ -230,7 +224,7 @@ public:
}
try {
DataStream ssValue{MakeByteSpan(*strValue)};
ssValue.Xor(obfuscate_key);
m_obfuscation(ssValue);
ssValue >> value;
} catch (const std::exception&) {
return false;

View file

@ -797,13 +797,13 @@ void BlockManager::UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const
AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const
{
return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_xor_key};
return AutoFile{m_block_file_seq.Open(pos, fReadOnly), m_obfuscation};
}
/** Open an undo file (rev?????.dat) */
AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const
{
return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_xor_key};
return AutoFile{m_undo_file_seq.Open(pos, fReadOnly), m_obfuscation};
}
fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const
@ -954,11 +954,11 @@ bool BlockManager::SaveBlockUndo(const CBlockUndo& blockundo, BlockValidationSta
if (block.GetUndoPos().IsNull()) {
FlatFilePos pos;
const uint32_t blockundo_size{static_cast<uint32_t>(GetSerializeSize(blockundo))};
if (!FindUndoPos(state, block.nFile, pos, UNDO_DATA_DISK_OVERHEAD + blockundo_size)) {
if (!FindUndoPos(state, block.nFile, pos, blockundo_size + UNDO_DATA_DISK_OVERHEAD)) {
LogError("%s: FindUndoPos failed\n", __func__);
return false;
}
AutoFile fileout{m_undo_file_seq.Open(pos, false), {}}; // We'll obfuscate ourselves
AutoFile fileout{m_undo_file_seq.Open(pos, false), 0}; // We'll obfuscate ourselves
if (fileout.IsNull()) {
LogError("%s: OpenUndoFile failed\n", __func__);
return FatalError(m_opts.notifications, state, _("Failed to write undo data."));
@ -968,7 +968,7 @@ bool BlockManager::SaveBlockUndo(const CBlockUndo& blockundo, BlockValidationSta
DataStream header;
header.reserve(BLOCK_SERIALIZATION_HEADER_SIZE);
header << GetParams().MessageStart() << blockundo_size;
util::Xor(header, m_xor_key, pos.nPos);
m_obfuscation(header, pos.nPos);
fileout.write(header);
}
pos.nPos += BLOCK_SERIALIZATION_HEADER_SIZE;
@ -980,7 +980,7 @@ bool BlockManager::SaveBlockUndo(const CBlockUndo& blockundo, BlockValidationSta
DataStream undo_data;
undo_data.reserve(blockundo_size + sizeof(uint256));
undo_data << blockundo << hasher.GetHash();
util::Xor(undo_data, m_xor_key, pos.nPos);
m_obfuscation(undo_data, pos.nPos);
fileout.write(undo_data);
}
@ -1117,7 +1117,7 @@ FlatFilePos BlockManager::SaveBlock(const CBlock& block, int nHeight)
LogError("%s: FindNextBlockPos failed\n", __func__);
return FlatFilePos();
}
AutoFile fileout{m_block_file_seq.Open(pos, false), {}}; // We'll obfuscate ourselves
AutoFile fileout{m_block_file_seq.Open(pos, false), 0}; // We'll obfuscate ourselves
if (fileout.IsNull()) {
LogError("%s: OpenBlockFile failed\n", __func__);
m_opts.notifications.fatalError(_("Failed to write block."));
@ -1128,7 +1128,7 @@ FlatFilePos BlockManager::SaveBlock(const CBlock& block, int nHeight)
DataStream header;
header.reserve(BLOCK_SERIALIZATION_HEADER_SIZE);
header << GetParams().MessageStart() << block_size;
util::Xor(header, m_xor_key, pos.nPos);
m_obfuscation(header, pos.nPos);
fileout.write(header);
}
pos.nPos += BLOCK_SERIALIZATION_HEADER_SIZE;
@ -1136,14 +1136,14 @@ FlatFilePos BlockManager::SaveBlock(const CBlock& block, int nHeight)
DataStream block_data;
block_data.reserve(block_size);
block_data << TX_WITH_WITNESS(block);
util::Xor(block_data, m_xor_key, pos.nPos);
m_obfuscation(block_data, pos.nPos);
fileout.write(block_data);
}
return pos;
}
static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
static Obfuscation InitBlocksdirXorKey(const BlockManager::Options& opts)
{
// Bytes are serialized without length indicator, so this is also the exact
// size of the XOR-key file.
@ -1180,12 +1180,12 @@ static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
};
}
LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(xor_key));
return std::vector<std::byte>{xor_key.begin(), xor_key.end()};
return Obfuscation{xor_key};
}
BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts)
: m_prune_mode{opts.prune_target > 0},
m_xor_key{InitBlocksdirXorKey(opts)},
m_obfuscation{InitBlocksdirXorKey(opts)},
m_opts{std::move(opts)},
m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}},
m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}},

View file

@ -235,7 +235,7 @@ private:
const bool m_prune_mode;
const std::vector<std::byte> m_xor_key;
const Obfuscation m_obfuscation;
/** Dirty block index entries. */
std::set<CBlockIndex*> m_dirty_blockindex;

View file

@ -58,15 +58,15 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active
try {
uint64_t version;
file >> version;
std::vector<std::byte> xor_key;
if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) {
// Leave XOR-key empty
file.SetObfuscation(0);
} else if (version == MEMPOOL_DUMP_VERSION) {
file >> xor_key;
Obfuscation obfuscation{0};
file >> obfuscation;
file.SetObfuscation(obfuscation);
} else {
return false;
}
file.SetXor(xor_key);
uint64_t total_txns_to_load;
file >> total_txns_to_load;
uint64_t txns_tried = 0;
@ -177,12 +177,13 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock
const uint64_t version{pool.m_opts.persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION};
file << version;
std::vector<std::byte> xor_key(8);
if (!pool.m_opts.persist_v1_dat) {
FastRandomContext{}.fillrand(xor_key);
file << xor_key;
const Obfuscation obfuscation{FastRandomContext{}.rand64()};
file << obfuscation;
file.SetObfuscation(obfuscation);
} else {
file.SetObfuscation(0);
}
file.SetXor(xor_key);
uint64_t mempool_transactions_to_write(vinfo.size());
file << mempool_transactions_to_write;

86
src/obfuscation.h Normal file
View file

@ -0,0 +1,86 @@
// Copyright (c) 2009-present The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef BITCOIN_OBFUSCATION_H
#define BITCOIN_OBFUSCATION_H
#include <array>
#include <cassert>
#include <cstdint>
#include <random>
#include <span.h>
#include <util/check.h>
#include <cstring>
#include <climits>
#include <serialize.h>
class Obfuscation
{
public:
static constexpr size_t SIZE_BYTES{sizeof(uint64_t)};
private:
std::array<uint64_t, SIZE_BYTES> rotations; // Cached key rotations
void SetRotations(const uint64_t key)
{
for (size_t i{0}; i < SIZE_BYTES; ++i)
{
size_t key_rotation_bits{CHAR_BIT * i};
if constexpr (std::endian::native == std::endian::big) key_rotation_bits *= -1;
rotations[i] = std::rotr(key, key_rotation_bits);
}
}
static uint64_t ToUint64(const Span<const std::byte> key_span)
{
uint64_t key{};
std::memcpy(&key, key_span.data(), SIZE_BYTES);
return key;
}
static void Xor(Span<std::byte> write, const uint64_t key, const size_t size)
{
assert(size <= write.size());
uint64_t raw{};
std::memcpy(&raw, write.data(), size);
raw ^= key;
std::memcpy(write.data(), &raw, size);
}
public:
Obfuscation(const uint64_t key) { SetRotations(key); }
Obfuscation(const Span<const std::byte> key_span) : Obfuscation(ToUint64(key_span)) {}
Obfuscation(const std::array<const std::byte, SIZE_BYTES>& key_arr) : Obfuscation(ToUint64(key_arr)) {}
Obfuscation(const std::vector<uint8_t>& key_vec) : Obfuscation(MakeByteSpan(key_vec)) {}
uint64_t Key() const { return rotations[0]; }
operator bool() const { return Key() != 0; }
void operator()(Span<std::byte> write, const size_t key_offset_bytes = 0) const
{
if (!*this) return;
const uint64_t rot_key{rotations[key_offset_bytes % SIZE_BYTES]}; // Continue obfuscation from where we left off
for (; write.size() >= SIZE_BYTES; write = write.subspan(SIZE_BYTES)) { // Process multiple bytes at a time
Xor(write, rot_key, SIZE_BYTES);
}
Xor(write, rot_key, write.size());
}
template <typename Stream>
void Serialize(Stream& s) const
{
std::vector<std::byte> bytes(SIZE_BYTES);
std::memcpy(bytes.data(), &rotations[0], SIZE_BYTES);
s << bytes;
}
template <typename Stream>
void Unserialize(Stream& s)
{
std::vector<std::byte> bytes(SIZE_BYTES);
s >> bytes;
SetRotations(ToUint64(bytes));
}
};
#endif // BITCOIN_OBFUSCATION_H

View file

@ -9,8 +9,7 @@
#include <array>
AutoFile::AutoFile(std::FILE* file, std::vector<std::byte> data_xor)
: m_file{file}, m_xor{std::move(data_xor)}
AutoFile::AutoFile(std::FILE* file, const Obfuscation& obfuscation) : m_file{file}, m_obfuscation{obfuscation}
{
if (!IsNull()) {
auto pos{std::ftell(m_file)};
@ -21,12 +20,12 @@ AutoFile::AutoFile(std::FILE* file, std::vector<std::byte> data_xor)
std::size_t AutoFile::detail_fread(Span<std::byte> dst)
{
if (!m_file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr");
size_t ret = std::fread(dst.data(), 1, dst.size(), m_file);
if (!m_xor.empty()) {
if (!m_position.has_value()) throw std::ios_base::failure("AutoFile::read: position unknown");
util::Xor(dst.subspan(0, ret), m_xor, *m_position);
const size_t ret = std::fread(dst.data(), 1, dst.size(), m_file);
if (m_obfuscation) {
if (!m_position) throw std::ios_base::failure("AutoFile::read: position unknown");
m_obfuscation(dst, *m_position);
}
if (m_position.has_value()) *m_position += ret;
if (m_position) *m_position += ret;
return ret;
}
@ -81,7 +80,7 @@ void AutoFile::ignore(size_t nSize)
void AutoFile::write(Span<const std::byte> src)
{
if (!m_file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr");
if (m_xor.empty()) {
if (!m_obfuscation) {
if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) {
throw std::ios_base::failure("AutoFile::write: write failed");
}
@ -91,8 +90,8 @@ void AutoFile::write(Span<const std::byte> src)
std::array<std::byte, 4096> buf;
while (src.size() > 0) {
auto buf_now{Span{buf}.first(std::min<size_t>(src.size(), buf.size()))};
std::copy(src.begin(), src.begin() + buf_now.size(), buf_now.begin());
util::Xor(buf_now, m_xor, *m_position);
std::copy_n(src.begin(), buf_now.size(), buf_now.begin());
m_obfuscation(buf_now, *m_position);
if (std::fwrite(buf_now.data(), 1, buf_now.size(), m_file) != buf_now.size()) {
throw std::ios_base::failure{"XorFile::write: failed"};
}

View file

@ -6,6 +6,7 @@
#ifndef BITCOIN_STREAMS_H
#define BITCOIN_STREAMS_H
#include <obfuscation.h>
#include <serialize.h>
#include <span.h>
#include <support/allocators/zeroafterfree.h>
@ -21,30 +22,8 @@
#include <stdint.h>
#include <string.h>
#include <string>
#include <utility>
#include <vector>
namespace util {
inline void Xor(Span<std::byte> write, Span<const std::byte> key, size_t key_offset = 0)
{
if (key.size() == 0) {
return;
}
key_offset %= key.size();
for (size_t i = 0, j = key_offset; i != write.size(); i++) {
write[i] ^= key[j++];
// This potentially acts on very many bytes of data, so it's
// important that we calculate `j`, i.e. the `key` index in this
// way instead of doing a %, which would effectively be a division
// for each byte Xor'd -- much slower than need be.
if (j == key.size())
j = 0;
}
}
} // namespace util
/* Minimal stream for overwriting and/or appending to an existing byte vector
*
* The referenced vector will grow as necessary
@ -261,21 +240,16 @@ public:
return (*this);
}
template<typename T>
template <typename T>
DataStream& operator>>(T&& obj)
{
::Unserialize(*this, obj);
return (*this);
}
/**
* XOR the contents of this stream with a certain key.
*
* @param[in] key The key used to XOR the data in this stream.
*/
void Xor(const std::vector<unsigned char>& key)
void Obfuscate(const Obfuscation& obfuscation)
{
util::Xor(MakeWritableByteSpan(*this), MakeByteSpan(key));
if (obfuscation) obfuscation(MakeWritableByteSpan(*this));
}
/** Compute total memory usage of this object (own memory + any dynamic memory). */
@ -392,11 +366,11 @@ class AutoFile
{
protected:
std::FILE* m_file;
std::vector<std::byte> m_xor;
Obfuscation m_obfuscation;
std::optional<int64_t> m_position;
public:
explicit AutoFile(std::FILE* file, std::vector<std::byte> data_xor={});
explicit AutoFile(std::FILE* file, const Obfuscation& obfuscation = 0);
~AutoFile() { fclose(); }
@ -428,7 +402,7 @@ public:
bool IsNull() const { return m_file == nullptr; }
/** Continue with a different XOR key */
void SetXor(std::vector<std::byte> data_xor) { m_xor = data_xor; }
void SetObfuscation(const Obfuscation& obfuscation) { m_obfuscation = obfuscation; }
/** Implementation detail, only used internally. */
std::size_t detail_fread(Span<std::byte> dst);

View file

@ -14,16 +14,6 @@
using util::ToString;
// Test if a string consists entirely of null characters
static bool is_null_key(const std::vector<unsigned char>& key) {
bool isnull = true;
for (unsigned int i = 0; i < key.size(); i++)
isnull &= (key[i] == '\x00');
return isnull;
}
BOOST_FIXTURE_TEST_SUITE(dbwrapper_tests, BasicTestingSetup)
BOOST_AUTO_TEST_CASE(dbwrapper)
@ -37,7 +27,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper)
uint256 res;
// Ensure that we're doing real obfuscation when obfuscate=true
BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscateKey(dbw)));
BOOST_CHECK(obfuscate == dbwrapper_private::GetObfuscation(dbw));
BOOST_CHECK(dbw.Write(key, in));
BOOST_CHECK(dbw.Read(key, res));
@ -57,7 +47,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_basic_data)
bool res_bool;
// Ensure that we're doing real obfuscation when obfuscate=true
BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscateKey(dbw)));
BOOST_CHECK(obfuscate == dbwrapper_private::GetObfuscation(dbw));
//Simulate block raw data - "b + block hash"
std::string key_block = "b" + m_rng.rand256().ToString();
@ -232,7 +222,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate)
BOOST_CHECK_EQUAL(res2.ToString(), in.ToString());
BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data
BOOST_CHECK(is_null_key(dbwrapper_private::GetObfuscateKey(odbw))); // The key should be an empty string
BOOST_CHECK(!dbwrapper_private::GetObfuscation(odbw));
uint256 in2 = m_rng.rand256();
uint256 res3;
@ -269,7 +259,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex)
// Check that the key/val we wrote with unobfuscated wrapper doesn't exist
uint256 res2;
BOOST_CHECK(!odbw.Read(key, res2));
BOOST_CHECK(!is_null_key(dbwrapper_private::GetObfuscateKey(odbw)));
BOOST_CHECK(dbwrapper_private::GetObfuscation(odbw));
uint256 in2 = m_rng.rand256();
uint256 res3;

View file

@ -20,7 +20,7 @@ FUZZ_TARGET(autofile)
FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider};
AutoFile auto_file{
fuzzed_file_provider.open(),
ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider),
fuzzed_data_provider.ConsumeIntegral<uint64_t>()
};
LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100)
{

View file

@ -22,7 +22,7 @@ FUZZ_TARGET(buffered_file)
std::optional<BufferedFile> opt_buffered_file;
AutoFile fuzzed_file{
fuzzed_file_provider.open(),
ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider),
fuzzed_data_provider.ConsumeIntegral<uint64_t>()
};
try {
auto n_buf_size = fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096);

View file

@ -14,34 +14,52 @@ using namespace std::string_literals;
BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup)
BOOST_AUTO_TEST_CASE(xor_roundtrip_random_chunks)
BOOST_AUTO_TEST_CASE(obfuscation_constructors)
{
auto apply_random_xor_chunks{[](std::span<std::byte> write, const std::span<std::byte> key, FastRandomContext& rng) {
for (size_t offset{0}; offset < write.size();) {
const size_t chunk_size{1 + rng.randrange(write.size() - offset)};
util::Xor(write.subspan(offset, chunk_size), key, offset);
offset += chunk_size;
}
}};
constexpr uint64_t test_key = 0x0123456789ABCDEF;
FastRandomContext rng{/*fDeterministic=*/false};
for (size_t test{0}; test < 100; ++test) {
const size_t write_size{1 + rng.randrange(100U)};
const std::vector original{rng.randbytes<std::byte>(write_size)};
std::vector roundtrip{original};
// Direct uint64_t constructor
const Obfuscation obf1{test_key};
BOOST_CHECK_EQUAL(obf1.Key(), test_key);
std::vector key_bytes{rng.randbytes<std::byte>(sizeof(uint64_t))};
uint64_t key;
std::memcpy(&key, key_bytes.data(), sizeof key);
// Span constructor
std::array<std::byte, Obfuscation::SIZE_BYTES> key_bytes{};
std::memcpy(key_bytes.data(), &test_key, Obfuscation::SIZE_BYTES);
const Obfuscation obf2{Span{key_bytes}};
BOOST_CHECK_EQUAL(obf2.Key(), test_key);
apply_random_xor_chunks(roundtrip, key_bytes, rng);
// std::array<std:byte> constructor
const Obfuscation obf3{key_bytes};
BOOST_CHECK_EQUAL(obf3.Key(), test_key);
const bool all_zero = (key == 0) || (HexStr(key_bytes).find_first_not_of('0') >= write_size * 2);
BOOST_CHECK_EQUAL(original != roundtrip, !all_zero);
// std::vector<uint8_t> constructor
std::vector<uint8_t> uchar_key(Obfuscation::SIZE_BYTES);
std::memcpy(uchar_key.data(), &test_key, uchar_key.size());
const Obfuscation obf4{uchar_key};
BOOST_CHECK_EQUAL(obf4.Key(), test_key);
}
apply_random_xor_chunks(roundtrip, key_bytes, rng);
BOOST_CHECK(original == roundtrip);
}
BOOST_AUTO_TEST_CASE(obfuscation_serialize)
{
const Obfuscation original{0xDEADBEEF};
// Serialize
DataStream ds;
ds << original;
BOOST_CHECK_EQUAL(ds.size(), 1 + Obfuscation::SIZE_BYTES); // serialized as a vector
// Deserialize
Obfuscation recovered{0};
ds >> recovered;
BOOST_CHECK_EQUAL(recovered.Key(), original.Key());
}
BOOST_AUTO_TEST_CASE(obfuscation_empty)
{
const Obfuscation null_obf{0};
BOOST_CHECK(!null_obf);
}
BOOST_AUTO_TEST_CASE(xor_bytes_reference)
@ -57,33 +75,60 @@ BOOST_AUTO_TEST_CASE(xor_bytes_reference)
const size_t write_size{1 + rng.randrange(100U)};
const size_t key_offset{rng.randrange(3 * 8U)}; // Should wrap around
std::vector key_bytes{rng.randbytes<std::byte>(sizeof(uint64_t))};
uint64_t key;
std::memcpy(&key, key_bytes.data(), sizeof key);
const auto key_bytes{rng.randbytes<std::byte>(Obfuscation::SIZE_BYTES)};
const Obfuscation obfuscation{key_bytes};
std::vector expected{rng.randbytes<std::byte>(write_size)};
std::vector actual{expected};
expected_xor(expected, key_bytes, key_offset);
util::Xor(actual, key_bytes, key_offset);
obfuscation(actual, key_offset);
BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(), actual.begin(), actual.end());
}
}
BOOST_AUTO_TEST_CASE(xor_roundtrip_random_chunks)
{
auto apply_random_xor_chunks{[](std::span<std::byte> write, const Obfuscation& obfuscation, FastRandomContext& rng) {
for (size_t offset{0}; offset < write.size();) {
const size_t chunk_size{1 + rng.randrange(write.size() - offset)};
obfuscation(write.subspan(offset, chunk_size), offset);
offset += chunk_size;
}
}};
FastRandomContext rng{/*fDeterministic=*/false};
for (size_t test{0}; test < 100; ++test) {
const size_t write_size{1 + rng.randrange(100U)};
const std::vector original{rng.randbytes<std::byte>(write_size)};
std::vector roundtrip{original};
const auto key_bytes{rng.randbytes<std::byte>(Obfuscation::SIZE_BYTES)};
const Obfuscation obfuscation{key_bytes};
apply_random_xor_chunks(roundtrip, obfuscation, rng);
const bool all_zero = !obfuscation || (HexStr(key_bytes).find_first_not_of('0') >= write_size * 2);
BOOST_CHECK_EQUAL(original != roundtrip, !all_zero);
apply_random_xor_chunks(roundtrip, obfuscation, rng);
BOOST_CHECK(original == roundtrip);
}
}
BOOST_AUTO_TEST_CASE(xor_file)
{
fs::path xor_path{m_args.GetDataDirBase() / "test_xor.bin"};
auto raw_file{[&](const auto& mode) { return fsbridge::fopen(xor_path, mode); }};
const std::vector<uint8_t> test1{1, 2, 3};
const std::vector<uint8_t> test2{4, 5};
const std::vector xor_pat{std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}};
constexpr std::array xor_pat{std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}, std::byte{0xff}, std::byte{0x00}};
uint64_t xor_key;
std::memcpy(&xor_key, xor_pat.data(), sizeof xor_key);
{
// Check errors for missing file
AutoFile xor_file{raw_file("rb"), xor_pat};
AutoFile xor_file{raw_file("rb"), xor_key};
BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullpt"});
BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullpt"});
BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullpt"});
@ -95,7 +140,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
#else
const char* mode = "wbx";
#endif
AutoFile xor_file{raw_file(mode), xor_pat};
AutoFile xor_file{raw_file(mode), xor_key};
xor_file << test1 << test2;
}
{
@ -108,7 +153,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
BOOST_CHECK_EXCEPTION(non_xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"});
}
{
AutoFile xor_file{raw_file("rb"), xor_pat};
AutoFile xor_file{raw_file("rb"), xor_key};
std::vector<std::byte> read1, read2;
xor_file >> read1 >> read2;
BOOST_CHECK_EQUAL(HexStr(read1), HexStr(test1));
@ -117,7 +162,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"});
}
{
AutoFile xor_file{raw_file("rb"), xor_pat};
AutoFile xor_file{raw_file("rb"), xor_key};
std::vector<std::byte> read2;
// Check that ignore works
xor_file.ignore(4);
@ -285,7 +330,7 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor)
// Degenerate case
{
DataStream ds{in};
ds.Xor({0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00});
Obfuscation{0}(ds);
BOOST_CHECK_EQUAL(""s, ds.str());
}
@ -293,12 +338,10 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor)
in.push_back(std::byte{0xf0});
{
const std::vector xor_pat{std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}};
uint64_t xor_key;
std::memcpy(&xor_key, xor_pat.data(), sizeof xor_key);
const Obfuscation obfuscation{{std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}, std::byte{0xff}}};
DataStream ds{in};
ds.Xor({0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff});
obfuscation(ds);
BOOST_CHECK_EQUAL("\xf0\x0f"s, ds.str());
}
@ -307,12 +350,10 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor)
in.push_back(std::byte{0x0f});
{
const std::vector xor_pat{std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}};
uint64_t xor_key;
std::memcpy(&xor_key, xor_pat.data(), sizeof xor_key);
const Obfuscation obfuscation{{std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}, std::byte{0xff}, std::byte{0x0f}}};
DataStream ds{in};
ds.Xor({0xff, 0x0f, 0xff, 0x0f, 0xff, 0x0f, 0xff, 0x0f});
obfuscation(ds);
BOOST_CHECK_EQUAL("\x0f\x00"s, ds.str());
}
}