optimization: migrate fixed-size obfuscation from std::vector<std::byte> to uint64_t

Since `util::Xor` now operates on `uint64_t` keys, migrated the obfuscation key end‑to‑end - from use sites in util::Xor up through streams, LevelDB wrappers, and disk (de)serialization - replacing all former `std::vector<std::byte>` keys with `uint64_t` (we still serialize them as vectors but convert immediately to `uint64_t` on load). This is why tests still generate vector keys and convert them to `uint64_t` later instead of generating them directly.

We also short‑circuit `Xor` calls when the key is zero to avoid unnecessary calculations (e.g., `MakeWritableByteSpan`).

In `Obfuscation::Unserialize` we can safely throw an `std::logic_error` since during mempool fuzzing `mempool_persist.cpp#L141` catches and ignored these errors safely.

>  cmake -B build -DBUILD_BENCH=ON -DCMAKE_BUILD_TYPE=Release \
&& cmake --build build -j$(nproc) \
&& build/bin/bench_bitcoin -filter='XorObfuscationBench' -min-time=10000

C++ compiler .......................... AppleClang 17.0.0.17000013

|              ns/MiB |               MiB/s |    err% |     total | benchmark
|--------------------:|--------------------:|--------:|----------:|:----------
|           14,730.40 |           67,886.80 |    0.1% |     11.01 | `XorObfuscationBench`

C++ compiler .......................... GNU 13.3.0

|              ns/MiB |               MiB/s |    err% |         ins/MiB |         cyc/MiB |    IPC |        bra/MiB |   miss% |     total | benchmark
|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------
|           51,187.17 |           19,536.15 |    0.0% |      327,683.95 |      183,747.58 |  1.783 |      65,536.55 |    0.0% |     11.00 | `XorObfuscationBench`

----

A few other benchmarks that seem to have improved as well (tested with Clang only):
Before:

|               ns/op |                op/s |    err% |     total | benchmark
|--------------------:|--------------------:|--------:|----------:|:----------
|        2,202,618.49 |              454.01 |    0.2% |     11.01 | `ReadBlockBench`
|          734,444.92 |            1,361.57 |    0.3% |     10.66 | `ReadRawBlockBench`

After:

|               ns/op |                op/s |    err% |     total | benchmark
|--------------------:|--------------------:|--------:|----------:|:----------
|        1,912,308.06 |              522.93 |    0.4% |     10.98 | `ReadBlockBench`
|           49,092.93 |           20,369.53 |    0.2% |     10.99 | `ReadRawBlockBench`

Co-authored-by: Hodlinator <172445034+hodlinator@users.noreply.github.com>
This commit is contained in:
Lőrinc 2024-12-21 16:15:04 +01:00
parent e80438d0e9
commit bba64732ff
13 changed files with 224 additions and 138 deletions

View file

@ -17,11 +17,11 @@ static void ObfuscationBench(benchmark::Bench& bench)
constexpr size_t bytes{10_MiB}; constexpr size_t bytes{10_MiB};
auto test_data{rng.randbytes<std::byte>(bytes)}; auto test_data{rng.randbytes<std::byte>(bytes)};
const std::vector obfuscation{rng.randbytes<std::byte>(8)}; const Obfuscation obfuscation{rng.rand64()};
size_t offset{0}; size_t offset{0};
bench.batch(bytes / 1_MiB).unit("MiB").run([&] { bench.batch(bytes / 1_MiB).unit("MiB").run([&] {
util::Obfuscation(test_data, obfuscation, offset++); obfuscation(test_data, offset++);
ankerl::nanobench::doNotOptimizeAway(test_data); ankerl::nanobench::doNotOptimizeAway(test_data);
}); });
} }

View file

@ -173,7 +173,7 @@ void CDBBatch::Clear()
void CDBBatch::WriteImpl(std::span<const std::byte> key, DataStream& ssValue) void CDBBatch::WriteImpl(std::span<const std::byte> key, DataStream& ssValue)
{ {
leveldb::Slice slKey(CharCast(key.data()), key.size()); leveldb::Slice slKey(CharCast(key.data()), key.size());
ssValue.Obfuscate(dbwrapper_private::GetObfuscation(parent)); dbwrapper_private::GetObfuscation(parent)(ssValue);
leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size()); leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size());
m_impl_batch->batch.Put(slKey, slValue); m_impl_batch->batch.Put(slKey, slValue);
} }
@ -249,21 +249,22 @@ CDBWrapper::CDBWrapper(const DBParams& params)
} }
{ {
m_obfuscation = std::vector<uint8_t>(OBFUSCATION_SIZE_BYTES, '\000'); // Needed for unobfuscated Read() below assert(m_obfuscation == 0); // Needed for unobfuscated Read() below
const bool key_missing{!Read(OBFUSCATION_KEY, m_obfuscation)}; std::vector<uint8_t> obfuscation_key_vector(Obfuscation::SIZE_BYTES, '\000');
const bool key_missing{!Read(OBFUSCATION_KEY, obfuscation_key_vector)};
if (key_missing && params.obfuscate && IsEmpty()) { if (key_missing && params.obfuscate && IsEmpty()) {
// Initialize non-degenerate obfuscation if it won't upset existing, non-obfuscated data. // Initialize non-degenerate obfuscation if it won't upset existing, non-obfuscated data.
std::vector<uint8_t> new_key(OBFUSCATION_SIZE_BYTES); std::vector<uint8_t> new_key(Obfuscation::SIZE_BYTES);
GetRandBytes(new_key); GetRandBytes(new_key);
// Write `new_key` so we don't obfuscate the key with itself // Write `new_key` so we don't obfuscate the key with itself
Write(OBFUSCATION_KEY, new_key); Write(OBFUSCATION_KEY, new_key);
m_obfuscation = std::move(new_key); obfuscation_key_vector = std::move(new_key);
LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(params.path), HexStr(m_obfuscation)); LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscation_key_vector));
} }
LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscation_key_vector));
LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(params.path), HexStr(m_obfuscation)); m_obfuscation = obfuscation_key_vector;
} }
} }
@ -309,19 +310,6 @@ size_t CDBWrapper::DynamicMemoryUsage() const
return parsed.value(); return parsed.value();
} }
const unsigned int CDBWrapper::OBFUSCATION_SIZE_BYTES = 8;
/**
* Returns a string (consisting of 8 random bytes) suitable for use as an
* obfuscating XOR key.
*/
std::vector<unsigned char> CDBWrapper::CreateObfuscation() const
{
std::vector<uint8_t> ret(OBFUSCATION_SIZE_BYTES);
GetRandBytes(ret);
return ret;
}
std::optional<std::string> CDBWrapper::ReadImpl(std::span<const std::byte> key) const std::optional<std::string> CDBWrapper::ReadImpl(std::span<const std::byte> key) const
{ {
leveldb::Slice slKey(CharCast(key.data()), key.size()); leveldb::Slice slKey(CharCast(key.data()), key.size());
@ -405,6 +393,6 @@ void CDBIterator::Next() { m_impl_iter->iter->Next(); }
namespace dbwrapper_private { namespace dbwrapper_private {
const std::vector<unsigned char>& GetObfuscation(const CDBWrapper &w) { return w.m_obfuscation; } Obfuscation GetObfuscation(const CDBWrapper& w) { return w.m_obfuscation; }
} // namespace dbwrapper_private } // namespace dbwrapper_private

View file

@ -18,7 +18,6 @@
#include <optional> #include <optional>
#include <stdexcept> #include <stdexcept>
#include <string> #include <string>
#include <vector>
static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64;
static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024; static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024;
@ -63,8 +62,7 @@ namespace dbwrapper_private {
* Database obfuscation should be considered an implementation detail of the * Database obfuscation should be considered an implementation detail of the
* specific database. * specific database.
*/ */
const std::vector<unsigned char>& GetObfuscation(const CDBWrapper &w); Obfuscation GetObfuscation(const CDBWrapper&);
}; // namespace dbwrapper_private }; // namespace dbwrapper_private
bool DestroyDB(const std::string& path_str); bool DestroyDB(const std::string& path_str);
@ -166,7 +164,7 @@ public:
template<typename V> bool GetValue(V& value) { template<typename V> bool GetValue(V& value) {
try { try {
DataStream ssValue{GetValueImpl()}; DataStream ssValue{GetValueImpl()};
ssValue.Obfuscate(dbwrapper_private::GetObfuscation(parent)); dbwrapper_private::GetObfuscation(parent)(ssValue);
ssValue >> value; ssValue >> value;
} catch (const std::exception&) { } catch (const std::exception&) {
return false; return false;
@ -179,7 +177,7 @@ struct LevelDBContext;
class CDBWrapper class CDBWrapper
{ {
friend const std::vector<unsigned char>& dbwrapper_private::GetObfuscation(const CDBWrapper &w); friend Obfuscation dbwrapper_private::GetObfuscation(const CDBWrapper&);
private: private:
//! holds all leveldb-specific fields of this class //! holds all leveldb-specific fields of this class
std::unique_ptr<LevelDBContext> m_db_context; std::unique_ptr<LevelDBContext> m_db_context;
@ -187,13 +185,8 @@ private:
//! the name of this database //! the name of this database
std::string m_name; std::string m_name;
//! a key used for optional XOR-obfuscation of the database //! optional XOR-obfuscation of the database
std::vector<unsigned char> m_obfuscation; Obfuscation m_obfuscation{0};
//! the length of the obfuscate key in number of bytes
static const unsigned int OBFUSCATION_SIZE_BYTES;
std::vector<unsigned char> CreateObfuscation() const;
//! path to filesystem storage //! path to filesystem storage
const fs::path m_path; const fs::path m_path;
@ -230,7 +223,7 @@ public:
} }
try { try {
DataStream ssValue{MakeByteSpan(*strValue)}; DataStream ssValue{MakeByteSpan(*strValue)};
ssValue.Obfuscate(m_obfuscation); m_obfuscation(ssValue);
ssValue >> value; ssValue >> value;
} catch (const std::exception&) { } catch (const std::exception&) {
return false; return false;

View file

@ -1107,7 +1107,7 @@ static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
{ {
// Bytes are serialized without length indicator, so this is also the exact // Bytes are serialized without length indicator, so this is also the exact
// size of the XOR-key file. // size of the XOR-key file.
std::array<std::byte, 8> xor_key{}; std::array<std::byte, Obfuscation::SIZE_BYTES> xor_key{};
// Consider this to be the first run if the blocksdir contains only hidden // Consider this to be the first run if the blocksdir contains only hidden
// files (those which start with a .). Checking for a fully-empty dir would // files (those which start with a .). Checking for a fully-empty dir would
@ -1152,7 +1152,7 @@ static auto InitBlocksdirXorKey(const BlockManager::Options& opts)
}; };
} }
LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(xor_key)); LogInfo("Using obfuscation key for blocksdir *.dat files (%s): '%s'\n", fs::PathToString(opts.blocks_dir), HexStr(xor_key));
return std::vector<std::byte>{xor_key.begin(), xor_key.end()}; return Obfuscation{xor_key};
} }
BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts) BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts)

View file

@ -235,7 +235,7 @@ private:
const bool m_prune_mode; const bool m_prune_mode;
const std::vector<std::byte> m_obfuscation; const Obfuscation m_obfuscation;
/** Dirty block index entries. */ /** Dirty block index entries. */
std::set<CBlockIndex*> m_dirty_blockindex; std::set<CBlockIndex*> m_dirty_blockindex;

View file

@ -60,9 +60,9 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active
file >> version; file >> version;
if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) { if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) {
file.SetObfuscation({}); file.SetObfuscation(0);
} else if (version == MEMPOOL_DUMP_VERSION) { } else if (version == MEMPOOL_DUMP_VERSION) {
std::vector<std::byte> obfuscation; Obfuscation obfuscation{0};
file >> obfuscation; file >> obfuscation;
file.SetObfuscation(obfuscation); file.SetObfuscation(obfuscation);
} else { } else {
@ -180,12 +180,11 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock
file << version; file << version;
if (!pool.m_opts.persist_v1_dat) { if (!pool.m_opts.persist_v1_dat) {
std::vector<std::byte> obfuscation(8); const Obfuscation obfuscation{FastRandomContext{}.rand64()};
FastRandomContext{}.fillrand(obfuscation);
file << obfuscation; file << obfuscation;
file.SetObfuscation(obfuscation); file.SetObfuscation(obfuscation);
} else { } else {
file.SetObfuscation({}); file.SetObfuscation(0);
} }
uint64_t mempool_transactions_to_write(vinfo.size()); uint64_t mempool_transactions_to_write(vinfo.size());

86
src/obfuscation.h Normal file
View file

@ -0,0 +1,86 @@
// Copyright (c) 2025-present The Bitcoin Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef BITCOIN_OBFUSCATION_H
#define BITCOIN_OBFUSCATION_H
#include <span.h>
#include <tinyformat.h>
#include <array>
#include <bit>
#include <climits>
#include <stdexcept>
class Obfuscation
{
public:
static constexpr size_t SIZE_BYTES{sizeof(uint64_t)};
Obfuscation(const uint64_t key) { SetRotations(key); }
Obfuscation(const std::span<const std::byte, SIZE_BYTES> key_span) : Obfuscation(ToUint64(key_span)) {}
Obfuscation(const std::vector<uint8_t>& key_vec) : Obfuscation(MakeByteSpan(key_vec).first<SIZE_BYTES>()) {}
Obfuscation(const std::vector<std::byte>& key_vec) : Obfuscation(std::span(key_vec).first<SIZE_BYTES>()) {}
uint64_t Key() const { return m_rotations[0]; }
operator bool() const { return Key() != 0; }
void operator()(std::span<std::byte> target, const size_t key_offset_bytes = 0) const
{
if (!*this) return;
const uint64_t rot_key{m_rotations[key_offset_bytes % SIZE_BYTES]}; // Continue obfuscation from where we left off
for (; target.size() >= SIZE_BYTES; target = target.subspan(SIZE_BYTES)) { // Process multiple bytes at a time
Xor(target, rot_key, SIZE_BYTES);
}
Xor(target, rot_key, target.size());
}
template <typename Stream>
void Serialize(Stream& s) const
{
// Use vector serialization for convenient compact size prefix.
std::vector<std::byte> bytes(SIZE_BYTES);
std::memcpy(bytes.data(), &m_rotations[0], SIZE_BYTES);
s << bytes;
}
template <typename Stream>
void Unserialize(Stream& s)
{
std::vector<std::byte> bytes(SIZE_BYTES);
s >> bytes;
if (bytes.size() != SIZE_BYTES) throw std::logic_error(strprintf("Obfuscation key size should be exactly %s bytes long", SIZE_BYTES));
SetRotations(ToUint64(std::span<std::byte, SIZE_BYTES>(bytes)));
}
private:
// Cached key rotations for different offsets.
std::array<uint64_t, SIZE_BYTES> m_rotations;
void SetRotations(const uint64_t key)
{
for (size_t i{0}; i < SIZE_BYTES; ++i) {
size_t key_rotation_bits{CHAR_BIT * i};
if constexpr (std::endian::native == std::endian::big) key_rotation_bits *= -1;
m_rotations[i] = std::rotr(key, key_rotation_bits);
}
}
static uint64_t ToUint64(const std::span<const std::byte, SIZE_BYTES> key_span)
{
uint64_t key{};
std::memcpy(&key, key_span.data(), SIZE_BYTES);
return key;
}
static void Xor(std::span<std::byte> target, const uint64_t key, const size_t size)
{
assert(size <= target.size());
uint64_t raw{};
std::memcpy(&raw, target.data(), size);
raw ^= key;
std::memcpy(target.data(), &raw, size);
}
};
#endif // BITCOIN_OBFUSCATION_H

View file

@ -9,8 +9,7 @@
#include <array> #include <array>
AutoFile::AutoFile(std::FILE* file, std::vector<std::byte> obfuscation) AutoFile::AutoFile(std::FILE* file, const Obfuscation& obfuscation) : m_file{file}, m_obfuscation{obfuscation}
: m_file{file}, m_obfuscation{std::move(obfuscation)}
{ {
if (!IsNull()) { if (!IsNull()) {
auto pos{std::ftell(m_file)}; auto pos{std::ftell(m_file)};
@ -21,12 +20,12 @@ AutoFile::AutoFile(std::FILE* file, std::vector<std::byte> obfuscation)
std::size_t AutoFile::detail_fread(std::span<std::byte> dst) std::size_t AutoFile::detail_fread(std::span<std::byte> dst)
{ {
if (!m_file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr"); if (!m_file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr");
size_t ret = std::fread(dst.data(), 1, dst.size(), m_file); const size_t ret = std::fread(dst.data(), 1, dst.size(), m_file);
if (!m_obfuscation.empty()) { if (m_obfuscation) {
if (!m_position.has_value()) throw std::ios_base::failure("AutoFile::read: position unknown"); if (!m_position) throw std::ios_base::failure("AutoFile::read: position unknown");
util::Obfuscation(dst.subspan(0, ret), m_obfuscation, *m_position); m_obfuscation(dst, *m_position);
} }
if (m_position.has_value()) *m_position += ret; if (m_position) *m_position += ret;
return ret; return ret;
} }
@ -81,7 +80,7 @@ void AutoFile::ignore(size_t nSize)
void AutoFile::write(std::span<const std::byte> src) void AutoFile::write(std::span<const std::byte> src)
{ {
if (!m_file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr"); if (!m_file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr");
if (m_obfuscation.empty()) { if (!m_obfuscation) {
if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) { if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) {
throw std::ios_base::failure("AutoFile::write: write failed"); throw std::ios_base::failure("AutoFile::write: write failed");
} }
@ -100,9 +99,9 @@ void AutoFile::write(std::span<const std::byte> src)
void AutoFile::write_buffer(std::span<std::byte> src) void AutoFile::write_buffer(std::span<std::byte> src)
{ {
if (!m_file) throw std::ios_base::failure("AutoFile::write_buffer: file handle is nullptr"); if (!m_file) throw std::ios_base::failure("AutoFile::write_buffer: file handle is nullptr");
if (m_obfuscation.size()) { if (m_obfuscation) {
if (!m_position) throw std::ios_base::failure("AutoFile::write_buffer: obfuscation position unknown"); if (!m_position) throw std::ios_base::failure("AutoFile::write_buffer: obfuscation position unknown");
util::Obfuscation(src, m_obfuscation, *m_position); // obfuscate in-place m_obfuscation(src, *m_position); // obfuscate in-place
} }
if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) { if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) {
throw std::ios_base::failure("AutoFile::write_buffer: write failed"); throw std::ios_base::failure("AutoFile::write_buffer: write failed");

View file

@ -6,6 +6,7 @@
#ifndef BITCOIN_STREAMS_H #ifndef BITCOIN_STREAMS_H
#define BITCOIN_STREAMS_H #define BITCOIN_STREAMS_H
#include <obfuscation.h>
#include <serialize.h> #include <serialize.h>
#include <span.h> #include <span.h>
#include <support/allocators/zeroafterfree.h> #include <support/allocators/zeroafterfree.h>
@ -21,30 +22,8 @@
#include <stdint.h> #include <stdint.h>
#include <string.h> #include <string.h>
#include <string> #include <string>
#include <utility>
#include <vector> #include <vector>
namespace util {
inline void Obfuscation(std::span<std::byte> write, std::span<const std::byte> key, size_t key_offset = 0)
{
if (key.size() == 0) {
return;
}
key_offset %= key.size();
for (size_t i = 0, j = key_offset; i != write.size(); i++) {
write[i] ^= key[j++];
// This potentially acts on very many bytes of data, so it's
// important that we calculate `j`, i.e. the `key` index in this
// way instead of doing a %, which would effectively be a division
// for each byte Xor'd -- much slower than need be.
if (j == key.size())
j = 0;
}
}
} // namespace util
/* Minimal stream for overwriting and/or appending to an existing byte vector /* Minimal stream for overwriting and/or appending to an existing byte vector
* *
* The referenced vector will grow as necessary * The referenced vector will grow as necessary
@ -261,21 +240,16 @@ public:
return (*this); return (*this);
} }
template<typename T> template <typename T>
DataStream& operator>>(T&& obj) DataStream& operator>>(T&& obj)
{ {
::Unserialize(*this, obj); ::Unserialize(*this, obj);
return (*this); return (*this);
} }
/** void Obfuscate(const Obfuscation& obfuscation)
* XOR the contents of this stream with a certain key.
*
* @param[in] obfuscation The key used to XOR the data in this stream.
*/
void Obfuscate(const std::vector<unsigned char>& obfuscation)
{ {
util::Obfuscation(MakeWritableByteSpan(*this), MakeByteSpan(obfuscation)); if (obfuscation) obfuscation(MakeWritableByteSpan(*this));
} }
/** Compute total memory usage of this object (own memory + any dynamic memory). */ /** Compute total memory usage of this object (own memory + any dynamic memory). */
@ -392,11 +366,11 @@ class AutoFile
{ {
protected: protected:
std::FILE* m_file; std::FILE* m_file;
std::vector<std::byte> m_obfuscation; Obfuscation m_obfuscation;
std::optional<int64_t> m_position; std::optional<int64_t> m_position;
public: public:
explicit AutoFile(std::FILE* file, std::vector<std::byte> obfuscation={}); explicit AutoFile(std::FILE* file, const Obfuscation& obfuscation = 0);
~AutoFile() { fclose(); } ~AutoFile() { fclose(); }
@ -428,7 +402,7 @@ public:
bool IsNull() const { return m_file == nullptr; } bool IsNull() const { return m_file == nullptr; }
/** Continue with a different XOR key */ /** Continue with a different XOR key */
void SetObfuscation(std::vector<std::byte> obfuscation) { m_obfuscation = obfuscation; } void SetObfuscation(const Obfuscation& obfuscation) { m_obfuscation = obfuscation; }
/** Implementation detail, only used internally. */ /** Implementation detail, only used internally. */
std::size_t detail_fread(std::span<std::byte> dst); std::size_t detail_fread(std::span<std::byte> dst);

View file

@ -9,21 +9,12 @@
#include <util/string.h> #include <util/string.h>
#include <memory> #include <memory>
#include <ranges>
#include <boost/test/unit_test.hpp> #include <boost/test/unit_test.hpp>
using util::ToString; using util::ToString;
// Test if a string consists entirely of null characters
static bool is_null_key(const std::vector<unsigned char>& key) {
bool isnull = true;
for (unsigned int i = 0; i < key.size(); i++)
isnull &= (key[i] == '\x00');
return isnull;
}
BOOST_FIXTURE_TEST_SUITE(dbwrapper_tests, BasicTestingSetup) BOOST_FIXTURE_TEST_SUITE(dbwrapper_tests, BasicTestingSetup)
BOOST_AUTO_TEST_CASE(dbwrapper) BOOST_AUTO_TEST_CASE(dbwrapper)
@ -33,7 +24,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper)
constexpr size_t CACHE_SIZE{1_MiB}; constexpr size_t CACHE_SIZE{1_MiB};
const fs::path path{m_args.GetDataDirBase() / "dbwrapper"}; const fs::path path{m_args.GetDataDirBase() / "dbwrapper"};
std::vector<uint8_t> obfuscation_key{}; uint64_t obfuscation_key{};
std::vector<std::pair<uint8_t, uint256>> key_values{}; std::vector<std::pair<uint8_t, uint256>> key_values{};
// Write values // Write values
@ -42,9 +33,9 @@ BOOST_AUTO_TEST_CASE(dbwrapper)
BOOST_CHECK_EQUAL(obfuscate, !dbw.IsEmpty()); BOOST_CHECK_EQUAL(obfuscate, !dbw.IsEmpty());
// Ensure that we're doing real obfuscation when obfuscate=true // Ensure that we're doing real obfuscation when obfuscate=true
BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscation(dbw))); BOOST_CHECK_EQUAL(obfuscate, dbwrapper_private::GetObfuscation(dbw));
obfuscation_key = dbwrapper_private::GetObfuscation(dbw); obfuscation_key = dbwrapper_private::GetObfuscation(dbw).Key();
for (uint8_t k{0}; k < 10; ++k) { for (uint8_t k{0}; k < 10; ++k) {
uint8_t key{k}; uint8_t key{k};
@ -57,7 +48,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper)
// Verify that the obfuscation key is never obfuscated // Verify that the obfuscation key is never obfuscated
{ {
CDBWrapper dbw{{.path = path, .cache_bytes = CACHE_SIZE, .obfuscate = false}}; CDBWrapper dbw{{.path = path, .cache_bytes = CACHE_SIZE, .obfuscate = false}};
BOOST_CHECK(obfuscation_key == dbwrapper_private::GetObfuscation(dbw)); BOOST_CHECK_EQUAL(obfuscation_key, dbwrapper_private::GetObfuscation(dbw).Key());
} }
// Read back the values // Read back the values
@ -65,8 +56,8 @@ BOOST_AUTO_TEST_CASE(dbwrapper)
CDBWrapper dbw{{.path = path, .cache_bytes = CACHE_SIZE, .obfuscate = obfuscate}}; CDBWrapper dbw{{.path = path, .cache_bytes = CACHE_SIZE, .obfuscate = obfuscate}};
// Ensure obfuscation is read back correctly // Ensure obfuscation is read back correctly
BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscation(dbw))); BOOST_CHECK_EQUAL(obfuscate, dbwrapper_private::GetObfuscation(dbw));
BOOST_CHECK(obfuscation_key == dbwrapper_private::GetObfuscation(dbw)); BOOST_CHECK_EQUAL(obfuscation_key, dbwrapper_private::GetObfuscation(dbw).Key());
// Verify all written values // Verify all written values
for (const auto& [key, expected_value] : key_values) { for (const auto& [key, expected_value] : key_values) {
@ -90,7 +81,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_basic_data)
bool res_bool; bool res_bool;
// Ensure that we're doing real obfuscation when obfuscate=true // Ensure that we're doing real obfuscation when obfuscate=true
BOOST_CHECK(obfuscate != is_null_key(dbwrapper_private::GetObfuscation(dbw))); BOOST_CHECK_EQUAL(obfuscate, dbwrapper_private::GetObfuscation(dbw));
//Simulate block raw data - "b + block hash" //Simulate block raw data - "b + block hash"
std::string key_block = "b" + m_rng.rand256().ToString(); std::string key_block = "b" + m_rng.rand256().ToString();
@ -149,7 +140,7 @@ BOOST_AUTO_TEST_CASE(dbwrapper_basic_data)
std::string file_option_tag = "F"; std::string file_option_tag = "F";
uint8_t filename_length = m_rng.randbits(8); uint8_t filename_length = m_rng.randbits(8);
std::string filename = "randomfilename"; std::string filename = "randomfilename";
std::string key_file_option = strprintf("%s%01x%s", file_option_tag,filename_length,filename); std::string key_file_option = strprintf("%s%01x%s", file_option_tag, filename_length, filename);
bool in_file_bool = m_rng.randbool(); bool in_file_bool = m_rng.randbool();
BOOST_CHECK(dbw.Write(key_file_option, in_file_bool)); BOOST_CHECK(dbw.Write(key_file_option, in_file_bool));
@ -265,7 +256,7 @@ BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate)
BOOST_CHECK_EQUAL(res2.ToString(), in.ToString()); BOOST_CHECK_EQUAL(res2.ToString(), in.ToString());
BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data
BOOST_CHECK(is_null_key(dbwrapper_private::GetObfuscation(odbw))); // The key should be an empty string BOOST_CHECK(!dbwrapper_private::GetObfuscation(odbw)); // The key should be an empty string
uint256 in2 = m_rng.rand256(); uint256 in2 = m_rng.rand256();
uint256 res3; uint256 res3;
@ -302,7 +293,7 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex)
// Check that the key/val we wrote with unobfuscated wrapper doesn't exist // Check that the key/val we wrote with unobfuscated wrapper doesn't exist
uint256 res2; uint256 res2;
BOOST_CHECK(!odbw.Read(key, res2)); BOOST_CHECK(!odbw.Read(key, res2));
BOOST_CHECK(!is_null_key(dbwrapper_private::GetObfuscation(odbw))); BOOST_CHECK(dbwrapper_private::GetObfuscation(odbw));
uint256 in2 = m_rng.rand256(); uint256 in2 = m_rng.rand256();
uint256 res3; uint256 res3;

View file

@ -20,7 +20,7 @@ FUZZ_TARGET(autofile)
FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider};
AutoFile auto_file{ AutoFile auto_file{
fuzzed_file_provider.open(), fuzzed_file_provider.open(),
ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<uint64_t>()
}; };
LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100)
{ {

View file

@ -22,7 +22,7 @@ FUZZ_TARGET(buffered_file)
std::optional<BufferedFile> opt_buffered_file; std::optional<BufferedFile> opt_buffered_file;
AutoFile fuzzed_file{ AutoFile fuzzed_file{
fuzzed_file_provider.open(), fuzzed_file_provider.open(),
ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<uint64_t>()
}; };
try { try {
auto n_buf_size = fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096); auto n_buf_size = fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096);

View file

@ -20,10 +20,10 @@ BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup)
// Test that obfuscation can be properly reverted even with random chunk sizes. // Test that obfuscation can be properly reverted even with random chunk sizes.
BOOST_AUTO_TEST_CASE(xor_roundtrip_random_chunks) BOOST_AUTO_TEST_CASE(xor_roundtrip_random_chunks)
{ {
auto apply_random_xor_chunks{[&](std::span<std::byte> target, const std::span<std::byte> obfuscation) { auto apply_random_xor_chunks{[&](std::span<std::byte> target, const Obfuscation& obfuscation) {
for (size_t offset{0}; offset < target.size();) { for (size_t offset{0}; offset < target.size();) {
const size_t chunk_size{1 + m_rng.randrange(target.size() - offset)}; const size_t chunk_size{1 + m_rng.randrange(target.size() - offset)};
util::Obfuscation(target.subspan(offset, chunk_size), obfuscation, offset); obfuscation(target.subspan(offset, chunk_size), offset);
offset += chunk_size; offset += chunk_size;
} }
}}; }};
@ -33,17 +33,16 @@ BOOST_AUTO_TEST_CASE(xor_roundtrip_random_chunks)
const std::vector original{m_rng.randbytes<std::byte>(write_size)}; const std::vector original{m_rng.randbytes<std::byte>(write_size)};
std::vector roundtrip{original}; std::vector roundtrip{original};
auto key_bytes{m_rng.randbool() ? std::vector(sizeof(uint64_t), std::byte{0}) : m_rng.randbytes<std::byte>(sizeof(uint64_t))}; const auto key_bytes{m_rng.randbool() ? std::vector(Obfuscation::SIZE_BYTES, std::byte{0}) : m_rng.randbytes<std::byte>(Obfuscation::SIZE_BYTES)};
uint64_t obfuscation; const Obfuscation obfuscation{key_bytes};
std::memcpy(&obfuscation, key_bytes.data(), sizeof(obfuscation)); apply_random_xor_chunks(roundtrip, obfuscation);
apply_random_xor_chunks(roundtrip, key_bytes);
// Verify intermediate state differs from original unless the key is all zeros // Verify intermediate state differs from original unless the key is all zeros
const bool all_zeros{(obfuscation == 0) || std::ranges::all_of( const bool all_zeros{!obfuscation || std::ranges::all_of(
std::span{key_bytes}.first(std::min(write_size, key_bytes.size())), [](auto b) { return b == std::byte{0}; })}; std::span{key_bytes}.first(std::min(write_size, Obfuscation::SIZE_BYTES)), [](auto b) { return b == std::byte{0}; })};
BOOST_CHECK_EQUAL(original != roundtrip, !all_zeros); BOOST_CHECK_EQUAL(original != roundtrip, !all_zeros);
apply_random_xor_chunks(roundtrip, key_bytes); apply_random_xor_chunks(roundtrip, obfuscation);
BOOST_CHECK(original == roundtrip); BOOST_CHECK(original == roundtrip);
} }
} }
@ -62,33 +61,90 @@ BOOST_AUTO_TEST_CASE(xor_bytes_reference)
const size_t write_size{1 + m_rng.randrange(100U)}; const size_t write_size{1 + m_rng.randrange(100U)};
const size_t key_offset{m_rng.randrange(3 * 8U)}; // Should wrap around const size_t key_offset{m_rng.randrange(3 * 8U)}; // Should wrap around
std::vector key_bytes{m_rng.randbytes<std::byte>(sizeof(uint64_t))}; const auto key_bytes{m_rng.randbytes<std::byte>(Obfuscation::SIZE_BYTES)};
uint64_t obfuscation; const Obfuscation obfuscation{key_bytes};
std::memcpy(&obfuscation, key_bytes.data(), sizeof(obfuscation));
std::vector expected{m_rng.randbytes<std::byte>(write_size)}; std::vector expected{m_rng.randbytes<std::byte>(write_size)};
std::vector actual{expected}; std::vector actual{expected};
expected_xor(expected, key_bytes, key_offset); expected_xor(expected, key_bytes, key_offset);
util::Obfuscation(actual, key_bytes, key_offset); obfuscation(actual, key_offset);
BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(), actual.begin(), actual.end()); BOOST_CHECK_EQUAL_COLLECTIONS(expected.begin(), expected.end(), actual.begin(), actual.end());
} }
} }
BOOST_AUTO_TEST_CASE(obfuscation_constructors)
{
constexpr uint64_t test_key{0x0123456789ABCDEF};
// Direct uint64_t constructor
{
const Obfuscation obfuscation{test_key};
BOOST_CHECK_EQUAL(obfuscation.Key(), test_key);
}
// std::span constructor
{
std::array<std::byte, Obfuscation::SIZE_BYTES> key_bytes{};
std::memcpy(key_bytes.data(), &test_key, Obfuscation::SIZE_BYTES);
const Obfuscation obfuscation{std::span{key_bytes}};
BOOST_CHECK_EQUAL(obfuscation.Key(), test_key);
}
// std::vector<uint8_t> constructor
{
std::vector<uint8_t> uint8_key(Obfuscation::SIZE_BYTES);
std::memcpy(uint8_key.data(), &test_key, uint8_key.size());
const Obfuscation obfuscation{uint8_key};
BOOST_CHECK_EQUAL(obfuscation.Key(), test_key);
}
// std::vector<std::byte> constructor
{
std::vector<std::byte> byte_vector_key(Obfuscation::SIZE_BYTES);
std::memcpy(byte_vector_key.data(), &test_key, byte_vector_key.size());
const Obfuscation obfuscation{byte_vector_key};
BOOST_CHECK_EQUAL(obfuscation.Key(), test_key);
}
}
BOOST_AUTO_TEST_CASE(obfuscation_serialize)
{
const Obfuscation original{0xDEADBEEF};
// Serialize
DataStream ds;
ds << original;
BOOST_CHECK_EQUAL(ds.size(), 1 + Obfuscation::SIZE_BYTES); // serialized as a vector
// Deserialize
Obfuscation recovered{0};
ds >> recovered;
BOOST_CHECK_EQUAL(recovered.Key(), original.Key());
}
BOOST_AUTO_TEST_CASE(obfuscation_empty)
{
const Obfuscation null_obf{0};
BOOST_CHECK(!null_obf);
}
BOOST_AUTO_TEST_CASE(xor_file) BOOST_AUTO_TEST_CASE(xor_file)
{ {
fs::path xor_path{m_args.GetDataDirBase() / "test_xor.bin"}; fs::path xor_path{m_args.GetDataDirBase() / "test_xor.bin"};
auto raw_file{[&](const auto& mode) { return fsbridge::fopen(xor_path, mode); }}; auto raw_file{[&](const auto& mode) { return fsbridge::fopen(xor_path, mode); }};
const std::vector<uint8_t> test1{1, 2, 3}; const std::vector<uint8_t> test1{1, 2, 3};
const std::vector<uint8_t> test2{4, 5}; const std::vector<uint8_t> test2{4, 5};
auto key_bytes{"ff00ff00ff00ff00"_hex_v}; auto key_bytes{"ff00ff00ff00ff00"_hex};
uint64_t xor_key; uint64_t xor_key;
std::memcpy(&xor_key, key_bytes.data(), sizeof(xor_key)); std::memcpy(&xor_key, key_bytes.data(), sizeof(xor_key));
{ {
// Check errors for missing file // Check errors for missing file
AutoFile xor_file{raw_file("rb"), key_bytes}; AutoFile xor_file{raw_file("rb"), {key_bytes}};
BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullpt"}); BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullpt"});
BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullpt"}); BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullpt"});
BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullpt"}); BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullpt"});
@ -100,7 +156,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
#else #else
const char* mode = "wbx"; const char* mode = "wbx";
#endif #endif
AutoFile xor_file{raw_file(mode), key_bytes}; AutoFile xor_file{raw_file(mode), xor_key};
xor_file << test1 << test2; xor_file << test1 << test2;
} }
{ {
@ -113,7 +169,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
BOOST_CHECK_EXCEPTION(non_xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"}); BOOST_CHECK_EXCEPTION(non_xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"});
} }
{ {
AutoFile xor_file{raw_file("rb"), key_bytes}; AutoFile xor_file{raw_file("rb"), xor_key};
std::vector<std::byte> read1, read2; std::vector<std::byte> read1, read2;
xor_file >> read1 >> read2; xor_file >> read1 >> read2;
BOOST_CHECK_EQUAL(HexStr(read1), HexStr(test1)); BOOST_CHECK_EQUAL(HexStr(read1), HexStr(test1));
@ -122,7 +178,7 @@ BOOST_AUTO_TEST_CASE(xor_file)
BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"}); BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"});
} }
{ {
AutoFile xor_file{raw_file("rb"), key_bytes}; AutoFile xor_file{raw_file("rb"), xor_key};
std::vector<std::byte> read2; std::vector<std::byte> read2;
// Check that ignore works // Check that ignore works
xor_file.ignore(4); xor_file.ignore(4);
@ -288,23 +344,23 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor)
// Degenerate case // Degenerate case
{ {
DataStream ds{}; DataStream ds{};
ds.Obfuscate("0000000000000000"_hex_v_u8); Obfuscation{0}(ds);
BOOST_CHECK_EQUAL(""s, ds.str()); BOOST_CHECK_EQUAL(""s, ds.str());
} }
{ {
const auto key_bytes{"ffffffffffffffff"_hex_v_u8}; const Obfuscation obfuscation{"ffffffffffffffff"_hex};
DataStream ds{"0ff0"_hex}; DataStream ds{"0ff0"_hex};
ds.Obfuscate(key_bytes); obfuscation(ds);
BOOST_CHECK_EQUAL("\xf0\x0f"s, ds.str()); BOOST_CHECK_EQUAL("\xf0\x0f"s, ds.str());
} }
{ {
const auto key_bytes{"ff0fff0fff0fff0f"_hex_v_u8}; const Obfuscation obfuscation{"ff0fff0fff0fff0f"_hex};
DataStream ds{"f00f"_hex}; DataStream ds{"f00f"_hex};
ds.Obfuscate(key_bytes); obfuscation(ds);
BOOST_CHECK_EQUAL("\x0f\x00"s, ds.str()); BOOST_CHECK_EQUAL("\x0f\x00"s, ds.str());
} }
} }
@ -617,7 +673,7 @@ BOOST_AUTO_TEST_CASE(buffered_reader_matches_autofile_random_content)
const FlatFilePos pos{0, 0}; const FlatFilePos pos{0, 0};
const FlatFileSeq test_file{m_args.GetDataDirBase(), "buffered_file_test_random", node::BLOCKFILE_CHUNK_SIZE}; const FlatFileSeq test_file{m_args.GetDataDirBase(), "buffered_file_test_random", node::BLOCKFILE_CHUNK_SIZE};
const std::vector obfuscation{m_rng.randbytes<std::byte>(8)}; const Obfuscation obfuscation{m_rng.rand64()};
// Write out the file with random content // Write out the file with random content
{ {
@ -670,7 +726,7 @@ BOOST_AUTO_TEST_CASE(buffered_writer_matches_autofile_random_content)
const FlatFileSeq test_buffered{m_args.GetDataDirBase(), "buffered_write_test", node::BLOCKFILE_CHUNK_SIZE}; const FlatFileSeq test_buffered{m_args.GetDataDirBase(), "buffered_write_test", node::BLOCKFILE_CHUNK_SIZE};
const FlatFileSeq test_direct{m_args.GetDataDirBase(), "direct_write_test", node::BLOCKFILE_CHUNK_SIZE}; const FlatFileSeq test_direct{m_args.GetDataDirBase(), "direct_write_test", node::BLOCKFILE_CHUNK_SIZE};
const std::vector obfuscation{m_rng.randbytes<std::byte>(8)}; const Obfuscation obfuscation{m_rng.rand64()};
{ {
DataBuffer test_data{m_rng.randbytes<std::byte>(file_size)}; DataBuffer test_data{m_rng.randbytes<std::byte>(file_size)};