From 05abf336f997f477c6f48412809ab540fccf1cb0 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 15 Nov 2024 14:15:12 -0500 Subject: [PATCH] txgraph: Add simulation fuzz test (tests) This adds a simulation fuzz test for txgraph, by comparing with a naive reimplementation that models the entire graph as a single DepGraph, and clusters in TxGraph as connected components within that DepGraph. --- src/test/fuzz/CMakeLists.txt | 1 + src/test/fuzz/txgraph.cpp | 420 +++++++++++++++++++++++++++++++++++ 2 files changed, 421 insertions(+) create mode 100644 src/test/fuzz/txgraph.cpp diff --git a/src/test/fuzz/CMakeLists.txt b/src/test/fuzz/CMakeLists.txt index e99c6d91f47..846afeeb474 100644 --- a/src/test/fuzz/CMakeLists.txt +++ b/src/test/fuzz/CMakeLists.txt @@ -124,6 +124,7 @@ add_executable(fuzz tx_in.cpp tx_out.cpp tx_pool.cpp + txgraph.cpp txorphan.cpp txrequest.cpp # Visual Studio 2022 version 17.12 introduced a bug diff --git a/src/test/fuzz/txgraph.cpp b/src/test/fuzz/txgraph.cpp new file mode 100644 index 00000000000..1d7fc8345a0 --- /dev/null +++ b/src/test/fuzz/txgraph.cpp @@ -0,0 +1,420 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace cluster_linearize; + +namespace { + +/** Data type representing a naive simulated TxGraph, keeping all transactions (even from + * disconnected components) in a single DepGraph. */ +struct SimTxGraph +{ + /** Maximum number of transactions to support simultaneously. Set this higher than txgraph's + * cluster count, so we can exercise situations with more transactions than fit in one + * cluster. */ + static constexpr unsigned MAX_TRANSACTIONS = CLUSTER_COUNT_LIMIT * 2; + /** Set type to use in the simulation. */ + using SetType = BitSet; + /** Data type for representing positions within SimTxGraph::graph. */ + using Pos = DepGraphIndex; + /** Constant to mean "missing in this graph". */ + static constexpr auto MISSING = Pos(-1); + + /** The dependency graph (for all transactions in the simulation, regardless of + * connectivity/clustering). */ + DepGraph graph; + /** For each position in graph, which TxGraph::Ref it corresponds with (if any). */ + std::array, MAX_TRANSACTIONS> simmap; + /** For each TxGraph::Ref in graph, the position it corresponds with. */ + std::map simrevmap; + /** The set of TxGraph::Ref entries that have been removed, but not yet destroyed. */ + std::vector> removed; + + /** Determine the number of (non-removed) transactions in the graph. */ + DepGraphIndex GetTransactionCount() const { return graph.TxCount(); } + + /** Get the position where ref occurs in this simulated graph, or -1 if it does not. */ + Pos Find(const TxGraph::Ref* ref) const + { + auto it = simrevmap.find(ref); + if (it != simrevmap.end()) return it->second; + return MISSING; + } + + /** Given a position in this simulated graph, get the corresponding TxGraph::Ref. */ + TxGraph::Ref* GetRef(Pos pos) + { + assert(graph.Positions()[pos]); + assert(simmap[pos]); + return simmap[pos].get(); + } + + /** Add a new transaction to the simulation. */ + TxGraph::Ref* AddTransaction(const FeePerWeight& feerate) + { + assert(graph.TxCount() < MAX_TRANSACTIONS); + auto simpos = graph.AddTransaction(feerate); + assert(graph.Positions()[simpos]); + simmap[simpos] = std::make_unique(); + auto ptr = simmap[simpos].get(); + simrevmap[ptr] = simpos; + return ptr; + } + + /** Add a dependency between two positions in this graph. */ + void AddDependency(TxGraph::Ref* parent, TxGraph::Ref* child) + { + auto par_pos = Find(parent); + if (par_pos == MISSING) return; + auto chl_pos = Find(child); + if (chl_pos == MISSING) return; + graph.AddDependencies(SetType::Singleton(par_pos), chl_pos); + } + + /** Modify the transaction fee of a ref, if it exists. */ + void SetTransactionFee(TxGraph::Ref* ref, int64_t fee) + { + auto pos = Find(ref); + if (pos == MISSING) return; + graph.FeeRate(pos).fee = fee; + } + + /** Remove the transaction in the specified position from the graph. */ + void RemoveTransaction(TxGraph::Ref* ref) + { + auto pos = Find(ref); + if (pos == MISSING) return; + graph.RemoveTransactions(SetType::Singleton(pos)); + simrevmap.erase(simmap[pos].get()); + // Retain the TxGraph::Ref corresponding to this position, so the Ref destruction isn't + // invoked until the simulation explicitly decided to do so. + removed.push_back(std::move(simmap[pos])); + simmap[pos].reset(); + } + + /** Construct the set with all positions in this graph corresponding to the specified + * TxGraph::Refs. All of them must occur in this graph and not be removed. */ + SetType MakeSet(std::span arg) + { + SetType ret; + for (TxGraph::Ref* ptr : arg) { + auto pos = Find(ptr); + assert(pos != Pos(-1)); + ret.Set(pos); + } + return ret; + } + + /** Get the set of ancestors (desc=false) or descendants (desc=true) in this graph. */ + SetType GetAncDesc(TxGraph::Ref* arg, bool desc) + { + auto pos = Find(arg); + if (pos == MISSING) return {}; + return desc ? graph.Descendants(pos) : graph.Ancestors(pos); + } + + /** Given a set of Refs (given as a vector of pointers), expand the set to include all its + * ancestors (desc=false) or all its descendants (desc=true) in this graph. */ + void IncludeAncDesc(std::vector& arg, bool desc) + { + std::vector ret; + for (auto ptr : arg) { + auto simpos = Find(ptr); + if (simpos != MISSING) { + for (auto i : desc ? graph.Descendants(simpos) : graph.Ancestors(simpos)) { + ret.push_back(simmap[i].get()); + } + } else { + ret.push_back(ptr); + } + } + // Deduplicate. + std::sort(ret.begin(), ret.end()); + ret.erase(std::unique(ret.begin(), ret.end()), ret.end()); + // Replace input. + arg = std::move(ret); + } +}; + +} // namespace + +FUZZ_TARGET(txgraph) +{ + // This is a big simulation test for TxGraph, which performs a fuzz-derived sequence of valid + // operations on a TxGraph instance, as well as on a simpler (mostly) reimplementation (see + // SimTxGraph above), comparing the outcome of functions that return a result, and finally + // performing a full comparison between the two. + + SeedRandomStateForTest(SeedRand::ZEROS); + FuzzedDataProvider provider(buffer.data(), buffer.size()); + + /** Internal test RNG, used only for decisions which would require significant amount of data + * to be read from the provider, without realistically impacting test sensitivity. */ + InsecureRandomContext rng(0xdecade2009added + buffer.size()); + + /** Variable used whenever an empty TxGraph::Ref is needed. */ + TxGraph::Ref empty_ref; + + // Construct a real and a simulated graph. + auto real = MakeTxGraph(); + SimTxGraph sim; + + /** Function to pick any Ref (from sim.simmap or sim.removed, or the empty Ref). */ + auto pick_fn = [&]() noexcept -> TxGraph::Ref* { + auto tx_count = sim.GetTransactionCount(); + /** The number of possible choices. */ + size_t choices = tx_count + sim.removed.size() + 1; + /** Pick one of them. */ + auto choice = provider.ConsumeIntegralInRange(0, choices - 1); + if (choice < tx_count) { + // Return from real. + for (auto i : sim.graph.Positions()) { + if (choice == 0) return sim.GetRef(i); + --choice; + } + assert(false); + } else { + choice -= tx_count; + } + if (choice < sim.removed.size()) { + // Return from removed. + return sim.removed[choice].get(); + } else { + choice -= sim.removed.size(); + } + // Return empty. + assert(choice == 0); + return &empty_ref; + }; + + LIMITED_WHILE(provider.remaining_bytes() > 0, 200) { + // Read a one-byte command. + int command = provider.ConsumeIntegral(); + // Treat it lowest bit as a flag (which selects a variant of some of the operations), and + // leave the rest of the bits in command. + bool alt = command & 1; + command >>= 1; + + // Keep decrementing command for each applicable operation, until one is hit. Multiple + // iterations may be necessary. + while (true) { + if (sim.GetTransactionCount() < SimTxGraph::MAX_TRANSACTIONS && command-- == 0) { + // AddTransaction. + int64_t fee; + int32_t size; + if (alt) { + // If alt is true, pick fee and size from the entire range. + fee = provider.ConsumeIntegralInRange(-0x8000000000000, 0x7ffffffffffff); + size = provider.ConsumeIntegralInRange(1, 0x3fffff); + } else { + // Otherwise, use smaller range which consume fewer fuzz input bytes, as just + // these are likely sufficient to trigger all interesting code paths already. + fee = provider.ConsumeIntegral(); + size = provider.ConsumeIntegral() + 1; + } + FeePerWeight feerate{fee, size}; + // Create a real TxGraph::Ref. + auto ref = real->AddTransaction(feerate); + // Create a unique_ptr place in the simulation to put the Ref in. + auto ref_loc = sim.AddTransaction(feerate); + // Move it in place. + *ref_loc = std::move(ref); + break; + } else if (sim.GetTransactionCount() + sim.removed.size() > 1 && command-- == 0) { + // AddDependency. + auto par = pick_fn(); + auto chl = pick_fn(); + auto pos_par = sim.Find(par); + auto pos_chl = sim.Find(chl); + if (pos_par != SimTxGraph::MISSING && pos_chl != SimTxGraph::MISSING) { + // Determine if adding this would introduce a cycle (not allowed by TxGraph), + // and if so, skip. + if (sim.graph.Ancestors(pos_par)[pos_chl]) break; + // Determine if adding this would violate CLUSTER_COUNT_LIMIT, and if so, skip. + auto temp_depgraph = sim.graph; + temp_depgraph.AddDependencies(SimTxGraph::SetType::Singleton(pos_par), pos_chl); + auto todo = temp_depgraph.Positions(); + bool oversize{false}; + while (todo.Any()) { + auto component = temp_depgraph.FindConnectedComponent(todo); + if (component.Count() > CLUSTER_COUNT_LIMIT) oversize = true; + todo -= component; + } + if (oversize) break; + } + sim.AddDependency(par, chl); + real->AddDependency(*par, *chl); + break; + } else if (sim.removed.size() < 100 && command-- == 0) { + // RemoveTransaction. Either all its ancestors or all its descendants are also + // removed (if any), to make sure TxGraph's reordering of removals and dependencies + // has no effect. + std::vector to_remove; + to_remove.push_back(pick_fn()); + sim.IncludeAncDesc(to_remove, alt); + // The order in which these ancestors/descendants are removed should not matter; + // randomly shuffle them. + std::shuffle(to_remove.begin(), to_remove.end(), rng); + for (TxGraph::Ref* ptr : to_remove) { + real->RemoveTransaction(*ptr); + sim.RemoveTransaction(ptr); + } + break; + } else if (sim.removed.size() > 0 && command-- == 0) { + // ~Ref. Destroying a TxGraph::Ref has an observable effect on the TxGraph it + // refers to, so this simulation permits doing so separately from other actions on + // TxGraph. + + // Pick a Ref of sim.removed to destroy. + auto removed_pos = provider.ConsumeIntegralInRange(0, sim.removed.size() - 1); + if (removed_pos != sim.removed.size() - 1) { + std::swap(sim.removed[removed_pos], sim.removed.back()); + } + sim.removed.pop_back(); + break; + } else if (command-- == 0) { + // SetTransactionFee. + int64_t fee; + if (alt) { + fee = provider.ConsumeIntegralInRange(-0x8000000000000, 0x7ffffffffffff); + } else { + fee = provider.ConsumeIntegral(); + } + auto ref = pick_fn(); + real->SetTransactionFee(*ref, fee); + sim.SetTransactionFee(ref, fee); + break; + } else if (command-- == 0) { + // GetTransactionCount. + assert(real->GetTransactionCount() == sim.GetTransactionCount()); + break; + } else if (command-- == 0) { + // Exists. + auto ref = pick_fn(); + bool exists = real->Exists(*ref); + bool should_exist = sim.Find(ref) != SimTxGraph::MISSING; + assert(exists == should_exist); + break; + } else if (command-- == 0) { + // GetIndividualFeerate. + auto ref = pick_fn(); + auto feerate = real->GetIndividualFeerate(*ref); + auto simpos = sim.Find(ref); + if (simpos == SimTxGraph::MISSING) { + assert(feerate.IsEmpty()); + } else { + assert(feerate == sim.graph.FeeRate(simpos)); + } + break; + } else if (command-- == 0) { + // GetAncestors/GetDescendants. + auto ref = pick_fn(); + auto result_set = sim.MakeSet(alt ? real->GetDescendants(*ref) : + real->GetAncestors(*ref)); + auto expect_set = sim.GetAncDesc(ref, alt); + assert(result_set == expect_set); + break; + } else if (command-- == 0) { + // GetCluster. + auto ref = pick_fn(); + auto result = real->GetCluster(*ref); + // Check cluster count limit. + assert(result.size() <= CLUSTER_COUNT_LIMIT); + // Require the result to be topologically valid and not contain duplicates. + auto left = sim.graph.Positions(); + for (auto refptr : result) { + auto simpos = sim.Find(refptr); + assert(simpos != SimTxGraph::MISSING); + assert(left[simpos]); + left.Reset(simpos); + assert(!sim.graph.Ancestors(simpos).Overlaps(left)); + } + // Require the set to be connected. + auto result_set = sim.MakeSet(result); + assert(sim.graph.IsConnected(result_set)); + // If ref exists, the result must contain it. If not, it must be empty. + auto simpos = sim.Find(ref); + if (simpos != SimTxGraph::MISSING) { + assert(result_set[simpos]); + } else { + assert(result_set.None()); + } + // Require the set not to have ancestors or descendants outside of it. + for (auto i : result_set) { + assert(sim.graph.Ancestors(i).IsSubsetOf(result_set)); + assert(sim.graph.Descendants(i).IsSubsetOf(result_set)); + } + break; + } + } + } + // Compare simple properties of the graph with the simulation. + assert(real->GetTransactionCount() == sim.GetTransactionCount()); + + // Perform a full comparison. + auto todo = sim.graph.Positions(); + // Iterate over all connected components of the resulting (simulated) graph, each of which + // should correspond to a cluster in the real one. + while (todo.Any()) { + auto component = sim.graph.FindConnectedComponent(todo); + todo -= component; + // Iterate over the transactions in that component. + for (auto i : component) { + // Check its individual feerate against simulation. + assert(sim.graph.FeeRate(i) == real->GetIndividualFeerate(*sim.GetRef(i))); + // Check its ancestors against simulation. + auto expect_anc = sim.graph.Ancestors(i); + auto anc = sim.MakeSet(real->GetAncestors(*sim.GetRef(i))); + assert(anc == expect_anc); + // Check its descendants against simulation. + auto expect_desc = sim.graph.Descendants(i); + auto desc = sim.MakeSet(real->GetDescendants(*sim.GetRef(i))); + assert(desc == expect_desc); + // Check the cluster the transaction is part of. + auto cluster = real->GetCluster(*sim.GetRef(i)); + assert(sim.MakeSet(cluster) == component); + // Check that the cluster is reported in a valid topological order (its + // linearization). + std::vector simlin; + SimTxGraph::SetType done; + for (TxGraph::Ref* ptr : cluster) { + auto simpos = sim.Find(ptr); + assert(sim.graph.Descendants(simpos).IsSubsetOf(component - done)); + done.Set(simpos); + assert(sim.graph.Ancestors(simpos).IsSubsetOf(done)); + simlin.push_back(simpos); + } + // Construct a chunking object for the simulated graph, using the reported cluster + // linearization as ordering. + cluster_linearize::LinearizationChunking simlinchunk(sim.graph, simlin); + for (unsigned chunknum = 0; chunknum < simlinchunk.NumChunksLeft(); ++chunknum) { + auto chunk = simlinchunk.GetChunk(chunknum); + // Require that the chunks of cluster linearizations are connected (this must + // be the case as all linearizations inside are PostLinearized). + assert(sim.graph.IsConnected(chunk.transactions)); + } + } + } + + // Remove all remaining transactions, because Refs cannot be destroyed otherwise (this will be + // addressed in a follow-up commit). + for (auto i : sim.graph.Positions()) { + auto ref = sim.GetRef(i); + real->RemoveTransaction(*ref); + } +}