phantom/bitcoin
1
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-04-29 14:59:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

txgraph: Add staging support (feature)

Browse source 
In order to make it easy to evaluate proposed changes to a TxGraph, introduce a
"staging" mode, where mutators (AddTransaction, AddDependency, RemoveTransaction)
do not modify the actual graph, but just a staging version of it. That staging
graph can then be commited (replacing the main one with it), or aborted (discarding
the staging).
This commit is contained in:
Pieter Wuille 2024-12-04 09:40:53 -05:00
parent c99c7300b4
commit 8c70688965
3 changed files with 863 additions and 383 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

333
src/test/fuzz/txgraph.cpp
View file

@ -13,6 +13,7 @@
#include <algorithm> #include <algorithm>
#include <map> #include <map>
#include <memory> #include <memory>
#include <set>
#include <stdint.h> #include <stdint.h>
#include <utility> #include <utility>
@ -21,7 +22,8 @@ using namespace cluster_linearize;
namespace { namespace {
/** Data type representing a naive simulated TxGraph, keeping all transactions (even from /** Data type representing a naive simulated TxGraph, keeping all transactions (even from
* disconnected components) in a single DepGraph. */ * disconnected components) in a single DepGraph. Unlike the real TxGraph, this only models
* a single graph, and multiple instances are used to simulate main/staging. */
struct SimTxGraph struct SimTxGraph
{ {
/** Maximum number of transactions to support simultaneously. Set this higher than txgraph's /** Maximum number of transactions to support simultaneously. Set this higher than txgraph's
@ -38,20 +40,28 @@ struct SimTxGraph
/** The dependency graph (for all transactions in the simulation, regardless of /** The dependency graph (for all transactions in the simulation, regardless of
* connectivity/clustering). */ * connectivity/clustering). */
DepGraph<SetType> graph; DepGraph<SetType> graph;
/** For each position in graph, which TxGraph::Ref it corresponds with (if any). */ /** For each position in graph, which TxGraph::Ref it corresponds with (if any). Use shared_ptr
std::array<std::unique_ptr<TxGraph::Ref>, MAX_TRANSACTIONS> simmap; * so that a SimTxGraph can be copied to create a staging one, while sharing Refs with
* the main graph. */
std::array<std::shared_ptr<TxGraph::Ref>, MAX_TRANSACTIONS> simmap;
/** For each TxGraph::Ref in graph, the position it corresponds with. */ /** For each TxGraph::Ref in graph, the position it corresponds with. */
std::map<const TxGraph::Ref*, Pos> simrevmap; std::map<const TxGraph::Ref*, Pos> simrevmap;
/** The set of TxGraph::Ref entries that have been removed, but not yet destroyed. */ /** The set of TxGraph::Ref entries that have been removed, but not yet destroyed. */
std::vector<std::unique_ptr<TxGraph::Ref>> removed; std::vector<std::shared_ptr<TxGraph::Ref>> removed;
/** Whether the graph is oversized (true = yes, false = no, std::nullopt = unknown). */ /** Whether the graph is oversized (true = yes, false = no, std::nullopt = unknown). */
std::optional<bool> oversized; std::optional<bool> oversized;
/** The configured maximum number of transactions per cluster. */ /** The configured maximum number of transactions per cluster. */
DepGraphIndex max_cluster_count; DepGraphIndex max_cluster_count;
/** Construct a new SimData with the specified maximum cluster count. */ /** Construct a new SimTxGraph with the specified maximum cluster count. */
explicit SimTxGraph(DepGraphIndex max_cluster) : max_cluster_count(max_cluster) {} explicit SimTxGraph(DepGraphIndex max_cluster) : max_cluster_count(max_cluster) {}
// Permit copying and moving.
SimTxGraph(const SimTxGraph&) noexcept = default;
SimTxGraph& operator=(const SimTxGraph&) noexcept = default;
SimTxGraph(SimTxGraph&&) noexcept = default;
SimTxGraph& operator=(SimTxGraph&&) noexcept = default;
/** Check whether this graph is oversized (contains a connected component whose number of /** Check whether this graph is oversized (contains a connected component whose number of
* transactions exceeds max_cluster_count. */ * transactions exceeds max_cluster_count. */
bool IsOversized() bool IsOversized()
@ -95,7 +105,7 @@ struct SimTxGraph
assert(graph.TxCount() < MAX_TRANSACTIONS); assert(graph.TxCount() < MAX_TRANSACTIONS);
auto simpos = graph.AddTransaction(feerate); auto simpos = graph.AddTransaction(feerate);
assert(graph.Positions()[simpos]); assert(graph.Positions()[simpos]);
simmap[simpos] = std::make_unique<TxGraph::Ref>(); simmap[simpos] = std::make_shared<TxGraph::Ref>();
auto ptr = simmap[simpos].get(); auto ptr = simmap[simpos].get();
simrevmap[ptr] = simpos; simrevmap[ptr] = simpos;
return ptr; return ptr;
@ -202,32 +212,43 @@ FUZZ_TARGET(txgraph)
// Decide the maximum number of transactions per cluster we will use in this simulation. // Decide the maximum number of transactions per cluster we will use in this simulation.
auto max_count = provider.ConsumeIntegralInRange<DepGraphIndex>(1, MAX_CLUSTER_COUNT_LIMIT); auto max_count = provider.ConsumeIntegralInRange<DepGraphIndex>(1, MAX_CLUSTER_COUNT_LIMIT);
// Construct a real and a simulated graph. // Construct a real graph, and a vector of simulated graphs (main, and possibly staging).
auto real = MakeTxGraph(max_count); auto real = MakeTxGraph(max_count);
SimTxGraph sim(max_count); std::vector<SimTxGraph> sims;
sims.reserve(2);
sims.emplace_back(max_count);
/** Function to pick any Ref (from sim.simmap or sim.removed, or the empty Ref). */ /** Function to pick any Ref (for either sim in sims: from sim.simmap or sim.removed, or the
* empty Ref). */
auto pick_fn = [&]() noexcept -> TxGraph::Ref* { auto pick_fn = [&]() noexcept -> TxGraph::Ref* {
auto tx_count = sim.GetTransactionCount(); size_t tx_count[2] = {sims[0].GetTransactionCount(), 0};
/** The number of possible choices. */ /** The number of possible choices. */
size_t choices = tx_count + sim.removed.size() + 1; size_t choices = tx_count[0] + sims[0].removed.size() + 1;
if (sims.size() == 2) {
tx_count[1] = sims[1].GetTransactionCount();
choices += tx_count[1] + sims[1].removed.size();
}
/** Pick one of them. */ /** Pick one of them. */
auto choice = provider.ConsumeIntegralInRange<size_t>(0, choices - 1); auto choice = provider.ConsumeIntegralInRange<size_t>(0, choices - 1);
if (choice < tx_count) { // Consider both main and (if it exists) staging.
// Return from real. for (size_t level = 0; level < sims.size(); ++level) {
for (auto i : sim.graph.Positions()) { auto& sim = sims[level];
if (choice == 0) return sim.GetRef(i); if (choice < tx_count[level]) {
--choice; // Return from graph.
for (auto i : sim.graph.Positions()) {
if (choice == 0) return sim.GetRef(i);
--choice;
}
assert(false);
} else {
choice -= tx_count[level];
}
if (choice < sim.removed.size()) {
// Return from removed.
return sim.removed[choice].get();
} else {
choice -= sim.removed.size();
} }
assert(false);
} else {
choice -= tx_count;
}
if (choice < sim.removed.size()) {
// Return from removed.
return sim.removed[choice].get();
} else {
choice -= sim.removed.size();
} }
// Return empty. // Return empty.
assert(choice == 0); assert(choice == 0);
@ -237,15 +258,24 @@ FUZZ_TARGET(txgraph)
LIMITED_WHILE(provider.remaining_bytes() > 0, 200) { LIMITED_WHILE(provider.remaining_bytes() > 0, 200) {
// Read a one-byte command. // Read a one-byte command.
int command = provider.ConsumeIntegral<uint8_t>(); int command = provider.ConsumeIntegral<uint8_t>();
// Treat it lowest bit as a flag (which selects a variant of some of the operations), and // Treat the lowest bit of a command as a flag (which selects a variant of some of the
// leave the rest of the bits in command. // operations), and the second-lowest bit as a way of selecting main vs. staging, and leave
// the rest of the bits in command.
bool alt = command & 1; bool alt = command & 1;
command >>= 1; bool use_main = command & 2;
command >>= 2;
// Provide convenient aliases for the top simulated graph (main, or staging if it exists),
// one for the simulated graph selected based on use_main (for operations that can operate
// on both graphs), and one that always refers to the main graph.
auto& top_sim = sims.back();
auto& sel_sim = use_main ? sims[0] : top_sim;
auto& main_sim = sims[0];
// Keep decrementing command for each applicable operation, until one is hit. Multiple // Keep decrementing command for each applicable operation, until one is hit. Multiple
// iterations may be necessary. // iterations may be necessary.
while (true) { while (true) {
if (sim.GetTransactionCount() < SimTxGraph::MAX_TRANSACTIONS && command-- == 0) { if (top_sim.GetTransactionCount() < SimTxGraph::MAX_TRANSACTIONS && command-- == 0) {
// AddTransaction. // AddTransaction.
int64_t fee; int64_t fee;
int32_t size; int32_t size;
@ -262,51 +292,54 @@ FUZZ_TARGET(txgraph)
FeePerWeight feerate{fee, size}; FeePerWeight feerate{fee, size};
// Create a real TxGraph::Ref. // Create a real TxGraph::Ref.
auto ref = real->AddTransaction(feerate); auto ref = real->AddTransaction(feerate);
// Create a unique_ptr place in the simulation to put the Ref in. // Create a shared_ptr place in the simulation to put the Ref in.
auto ref_loc = sim.AddTransaction(feerate); auto ref_loc = top_sim.AddTransaction(feerate);
// Move it in place. // Move it in place.
*ref_loc = std::move(ref); *ref_loc = std::move(ref);
break; break;
} else if (sim.GetTransactionCount() + sim.removed.size() > 1 && command-- == 0) { } else if (top_sim.GetTransactionCount() + top_sim.removed.size() > 1 && command-- == 0) {
// AddDependency. // AddDependency.
auto par = pick_fn(); auto par = pick_fn();
auto chl = pick_fn(); auto chl = pick_fn();
auto pos_par = sim.Find(par); auto pos_par = top_sim.Find(par);
auto pos_chl = sim.Find(chl); auto pos_chl = top_sim.Find(chl);
if (pos_par != SimTxGraph::MISSING && pos_chl != SimTxGraph::MISSING) { if (pos_par != SimTxGraph::MISSING && pos_chl != SimTxGraph::MISSING) {
// Determine if adding this would introduce a cycle (not allowed by TxGraph), // Determine if adding this would introduce a cycle (not allowed by TxGraph),
// and if so, skip. // and if so, skip.
if (sim.graph.Ancestors(pos_par)[pos_chl]) break; if (top_sim.graph.Ancestors(pos_par)[pos_chl]) break;
} }
sim.AddDependency(par, chl); top_sim.AddDependency(par, chl);
real->AddDependency(*par, *chl); real->AddDependency(*par, *chl);
break; break;
} else if (sim.removed.size() < 100 && command-- == 0) { } else if (top_sim.removed.size() < 100 && command-- == 0) {
// RemoveTransaction. Either all its ancestors or all its descendants are also // RemoveTransaction. Either all its ancestors or all its descendants are also
// removed (if any), to make sure TxGraph's reordering of removals and dependencies // removed (if any), to make sure TxGraph's reordering of removals and dependencies
// has no effect. // has no effect.
std::vector<TxGraph::Ref*> to_remove; std::vector<TxGraph::Ref*> to_remove;
to_remove.push_back(pick_fn()); to_remove.push_back(pick_fn());
sim.IncludeAncDesc(to_remove, alt); top_sim.IncludeAncDesc(to_remove, alt);
// The order in which these ancestors/descendants are removed should not matter; // The order in which these ancestors/descendants are removed should not matter;
// randomly shuffle them. // randomly shuffle them.
std::shuffle(to_remove.begin(), to_remove.end(), rng); std::shuffle(to_remove.begin(), to_remove.end(), rng);
for (TxGraph::Ref* ptr : to_remove) { for (TxGraph::Ref* ptr : to_remove) {
real->RemoveTransaction(*ptr); real->RemoveTransaction(*ptr);
sim.RemoveTransaction(ptr); top_sim.RemoveTransaction(ptr);
} }
break; break;
} else if (sim.removed.size() > 0 && command-- == 0) { } else if (sel_sim.removed.size() > 0 && command-- == 0) {
// ~Ref. Destroying a TxGraph::Ref has an observable effect on the TxGraph it // ~Ref. Destroying a TxGraph::Ref has an observable effect on the TxGraph it
// refers to, so this simulation permits doing so separately from other actions on // refers to, so this simulation permits doing so separately from other actions on
// TxGraph. // TxGraph.
// Pick a Ref of sim.removed to destroy. // Pick a Ref of sel_sim.removed to destroy. Note that the same Ref may still occur
auto removed_pos = provider.ConsumeIntegralInRange<size_t>(0, sim.removed.size() - 1); // in the other graph, and thus not actually trigger ~Ref yet (which is exactly
if (removed_pos != sim.removed.size() - 1) { // what we want, as destroying Refs is only allowed when it does not refer to an
std::swap(sim.removed[removed_pos], sim.removed.back()); // existing transaction in either graph).
auto removed_pos = provider.ConsumeIntegralInRange<size_t>(0, sel_sim.removed.size() - 1);
if (removed_pos != sel_sim.removed.size() - 1) {
std::swap(sel_sim.removed[removed_pos], sel_sim.removed.back());
} }
sim.removed.pop_back(); sel_sim.removed.pop_back();
break; break;
} else if (command-- == 0) { } else if (command-- == 0) {
// SetTransactionFee. // SetTransactionFee.
@ -318,77 +351,83 @@ FUZZ_TARGET(txgraph)
} }
auto ref = pick_fn(); auto ref = pick_fn();
real->SetTransactionFee(*ref, fee); real->SetTransactionFee(*ref, fee);
sim.SetTransactionFee(ref, fee); for (auto& sim : sims) {
sim.SetTransactionFee(ref, fee);
}
break; break;
} else if (command-- == 0) { } else if (command-- == 0) {
// GetTransactionCount. // GetTransactionCount.
assert(real->GetTransactionCount() == sim.GetTransactionCount()); assert(real->GetTransactionCount(use_main) == sel_sim.GetTransactionCount());
break; break;
} else if (command-- == 0) { } else if (command-- == 0) {
// Exists. // Exists.
auto ref = pick_fn(); auto ref = pick_fn();
bool exists = real->Exists(*ref); bool exists = real->Exists(*ref, use_main);
bool should_exist = sim.Find(ref) != SimTxGraph::MISSING; bool should_exist = sel_sim.Find(ref) != SimTxGraph::MISSING;
assert(exists == should_exist); assert(exists == should_exist);
break; break;
} else if (command-- == 0) { } else if (command-- == 0) {
// IsOversized. // IsOversized.
assert(sim.IsOversized() == real->IsOversized()); assert(sel_sim.IsOversized() == real->IsOversized(use_main));
break; break;
} else if (command-- == 0) { } else if (command-- == 0) {
// GetIndividualFeerate. // GetIndividualFeerate.
auto ref = pick_fn(); auto ref = pick_fn();
auto feerate = real->GetIndividualFeerate(*ref); auto feerate = real->GetIndividualFeerate(*ref);
auto simpos = sim.Find(ref); bool found{false};
if (simpos == SimTxGraph::MISSING) { for (auto& sim : sims) {
assert(feerate.IsEmpty()); auto simpos = sim.Find(ref);
} else { if (simpos != SimTxGraph::MISSING) {
assert(feerate == sim.graph.FeeRate(simpos)); found = true;
assert(feerate == sim.graph.FeeRate(simpos));
}
} }
if (!found) assert(feerate.IsEmpty());
break; break;
} else if (!sim.IsOversized() && command-- == 0) { } else if (!main_sim.IsOversized() && command-- == 0) {
// GetChunkFeerate. // GetMainChunkFeerate.
auto ref = pick_fn(); auto ref = pick_fn();
auto feerate = real->GetChunkFeerate(*ref); auto feerate = real->GetMainChunkFeerate(*ref);
auto simpos = sim.Find(ref); auto simpos = main_sim.Find(ref);
if (simpos == SimTxGraph::MISSING) { if (simpos == SimTxGraph::MISSING) {
assert(feerate.IsEmpty()); assert(feerate.IsEmpty());
} else { } else {
// Just do some quick checks that the reported value is in range. A full // Just do some quick checks that the reported value is in range. A full
// recomputation of expected chunk feerates is done at the end. // recomputation of expected chunk feerates is done at the end.
assert(feerate.size >= sim.graph.FeeRate(simpos).size); assert(feerate.size >= main_sim.graph.FeeRate(simpos).size);
} }
break; break;
} else if (!sim.IsOversized() && command-- == 0) { } else if (!sel_sim.IsOversized() && command-- == 0) {
// GetAncestors/GetDescendants. // GetAncestors/GetDescendants.
auto ref = pick_fn(); auto ref = pick_fn();
auto result = alt ? real->GetDescendants(*ref) : real->GetAncestors(*ref); auto result = alt ? real->GetDescendants(*ref, use_main)
: real->GetAncestors(*ref, use_main);
assert(result.size() <= max_count); assert(result.size() <= max_count);
auto result_set = sim.MakeSet(result); auto result_set = sel_sim.MakeSet(result);
assert(result.size() == result_set.Count()); assert(result.size() == result_set.Count());
auto expect_set = sim.GetAncDesc(ref, alt); auto expect_set = sel_sim.GetAncDesc(ref, alt);
assert(result_set == expect_set); assert(result_set == expect_set);
break; break;
} else if (!sim.IsOversized() && command-- == 0) { } else if (!sel_sim.IsOversized() && command-- == 0) {
// GetCluster. // GetCluster.
auto ref = pick_fn(); auto ref = pick_fn();
auto result = real->GetCluster(*ref); auto result = real->GetCluster(*ref, use_main);
// Check cluster count limit. // Check cluster count limit.
assert(result.size() <= max_count); assert(result.size() <= max_count);
// Require the result to be topologically valid and not contain duplicates. // Require the result to be topologically valid and not contain duplicates.
auto left = sim.graph.Positions(); auto left = sel_sim.graph.Positions();
for (auto refptr : result) { for (auto refptr : result) {
auto simpos = sim.Find(refptr); auto simpos = sel_sim.Find(refptr);
assert(simpos != SimTxGraph::MISSING); assert(simpos != SimTxGraph::MISSING);
assert(left[simpos]); assert(left[simpos]);
left.Reset(simpos); left.Reset(simpos);
assert(!sim.graph.Ancestors(simpos).Overlaps(left)); assert(!sel_sim.graph.Ancestors(simpos).Overlaps(left));
} }
// Require the set to be connected. // Require the set to be connected.
auto result_set = sim.MakeSet(result); auto result_set = sel_sim.MakeSet(result);
assert(sim.graph.IsConnected(result_set)); assert(sel_sim.graph.IsConnected(result_set));
// If ref exists, the result must contain it. If not, it must be empty. // If ref exists, the result must contain it. If not, it must be empty.
auto simpos = sim.Find(ref); auto simpos = sel_sim.Find(ref);
if (simpos != SimTxGraph::MISSING) { if (simpos != SimTxGraph::MISSING) {
assert(result_set[simpos]); assert(result_set[simpos]);
} else { } else {
@ -396,10 +435,29 @@ FUZZ_TARGET(txgraph)
} }
// Require the set not to have ancestors or descendants outside of it. // Require the set not to have ancestors or descendants outside of it.
for (auto i : result_set) { for (auto i : result_set) {
assert(sim.graph.Ancestors(i).IsSubsetOf(result_set)); assert(sel_sim.graph.Ancestors(i).IsSubsetOf(result_set));
assert(sim.graph.Descendants(i).IsSubsetOf(result_set)); assert(sel_sim.graph.Descendants(i).IsSubsetOf(result_set));
} }
break; break;
} else if (command-- == 0) {
// HaveStaging.
assert((sims.size() == 2) == real->HaveStaging());
break;
} else if (sims.size() < 2 && command-- == 0) {
// StartStaging.
sims.emplace_back(sims.back());
real->StartStaging();
break;
} else if (sims.size() > 1 && command-- == 0) {
// CommitStaging.
real->CommitStaging();
sims.erase(sims.begin());
break;
} else if (sims.size() > 1 && command-- == 0) {
// AbortStaging.
real->AbortStaging();
sims.pop_back();
break;
} }
} }
} }
@ -407,63 +465,70 @@ FUZZ_TARGET(txgraph)
// After running all modifications, perform an internal sanity check (before invoking // After running all modifications, perform an internal sanity check (before invoking
// inspectors that may modify the internal state). // inspectors that may modify the internal state).
real->SanityCheck(); real->SanityCheck();
assert(real->HaveStaging() == (sims.size() > 1));
// Compare simple properties of the graph with the simulation. // Try to run a full comparison, for both main_only=false and main_only=true in TxGraph
assert(real->IsOversized() == sim.IsOversized()); // inspector functions that support both.
assert(real->GetTransactionCount() == sim.GetTransactionCount()); for (int main_only = 0; main_only < 2; ++main_only) {
auto& sim = main_only ? sims[0] : sims.back();
// If the graph (and the simulation) are not oversized, perform a full comparison. // Compare simple properties of the graph with the simulation.
if (!sim.IsOversized()) { assert(real->IsOversized(main_only) == sim.IsOversized());
auto todo = sim.graph.Positions(); assert(real->GetTransactionCount(main_only) == sim.GetTransactionCount());
// Iterate over all connected components of the resulting (simulated) graph, each of which // If the graph (and the simulation) are not oversized, perform a full comparison.
// should correspond to a cluster in the real one. if (!sim.IsOversized()) {
while (todo.Any()) { auto todo = sim.graph.Positions();
auto component = sim.graph.FindConnectedComponent(todo); // Iterate over all connected components of the resulting (simulated) graph, each of which
todo -= component; // should correspond to a cluster in the real one.
// Iterate over the transactions in that component. while (todo.Any()) {
for (auto i : component) { auto component = sim.graph.FindConnectedComponent(todo);
// Check its individual feerate against simulation. todo -= component;
assert(sim.graph.FeeRate(i) == real->GetIndividualFeerate(*sim.GetRef(i))); // Iterate over the transactions in that component.
// Check its ancestors against simulation. for (auto i : component) {
auto expect_anc = sim.graph.Ancestors(i); // Check its individual feerate against simulation.
auto anc = sim.MakeSet(real->GetAncestors(*sim.GetRef(i))); assert(sim.graph.FeeRate(i) == real->GetIndividualFeerate(*sim.GetRef(i)));
assert(anc.Count() <= max_count); // Check its ancestors against simulation.
assert(anc == expect_anc); auto expect_anc = sim.graph.Ancestors(i);
// Check its descendants against simulation. auto anc = sim.MakeSet(real->GetAncestors(*sim.GetRef(i), main_only));
auto expect_desc = sim.graph.Descendants(i); assert(anc.Count() <= max_count);
auto desc = sim.MakeSet(real->GetDescendants(*sim.GetRef(i))); assert(anc == expect_anc);
assert(desc.Count() <= max_count); // Check its descendants against simulation.
assert(desc == expect_desc); auto expect_desc = sim.graph.Descendants(i);
// Check the cluster the transaction is part of. auto desc = sim.MakeSet(real->GetDescendants(*sim.GetRef(i), main_only));
auto cluster = real->GetCluster(*sim.GetRef(i)); assert(desc.Count() <= max_count);
assert(cluster.size() <= max_count); assert(desc == expect_desc);
assert(sim.MakeSet(cluster) == component); // Check the cluster the transaction is part of.
// Check that the cluster is reported in a valid topological order (its auto cluster = real->GetCluster(*sim.GetRef(i), main_only);
// linearization). assert(cluster.size() <= max_count);
std::vector<DepGraphIndex> simlin; assert(sim.MakeSet(cluster) == component);
SimTxGraph::SetType done; // Check that the cluster is reported in a valid topological order (its
for (TxGraph::Ref* ptr : cluster) { // linearization).
auto simpos = sim.Find(ptr); std::vector<DepGraphIndex> simlin;
assert(sim.graph.Descendants(simpos).IsSubsetOf(component - done)); SimTxGraph::SetType done;
done.Set(simpos); for (TxGraph::Ref* ptr : cluster) {
assert(sim.graph.Ancestors(simpos).IsSubsetOf(done)); auto simpos = sim.Find(ptr);
simlin.push_back(simpos); assert(sim.graph.Descendants(simpos).IsSubsetOf(component - done));
} done.Set(simpos);
// Construct a chunking object for the simulated graph, using the reported cluster assert(sim.graph.Ancestors(simpos).IsSubsetOf(done));
// linearization as ordering, and compare it against the reported chunk feerates. simlin.push_back(simpos);
cluster_linearize::LinearizationChunking simlinchunk(sim.graph, simlin); }
DepGraphIndex idx{0}; // Construct a chunking object for the simulated graph, using the reported cluster
for (unsigned chunknum = 0; chunknum < simlinchunk.NumChunksLeft(); ++chunknum) { // linearization as ordering, and compare it against the reported chunk feerates.
auto chunk = simlinchunk.GetChunk(chunknum); if (sims.size() == 1 || main_only) {
// Require that the chunks of cluster linearizations are connected (this must cluster_linearize::LinearizationChunking simlinchunk(sim.graph, simlin);
// be the case as all linearizations inside are PostLinearized). DepGraphIndex idx{0};
assert(sim.graph.IsConnected(chunk.transactions)); for (unsigned chunknum = 0; chunknum < simlinchunk.NumChunksLeft(); ++chunknum) {
// Check the chunk feerates of all transactions in the cluster. auto chunk = simlinchunk.GetChunk(chunknum);
while (chunk.transactions.Any()) { // Require that the chunks of cluster linearizations are connected (this must
assert(chunk.transactions[simlin[idx]]); // be the case as all linearizations inside are PostLinearized).
chunk.transactions.Reset(simlin[idx]); assert(sim.graph.IsConnected(chunk.transactions));
assert(chunk.feerate == real->GetChunkFeerate(*cluster[idx])); // Check the chunk feerates of all transactions in the cluster.
++idx; while (chunk.transactions.Any()) {
assert(chunk.transactions[simlin[idx]]);
chunk.transactions.Reset(simlin[idx]);
assert(chunk.feerate == real->GetMainChunkFeerate(*cluster[idx]));
++idx;
}
}
} }
} }
} }
@ -475,8 +540,10 @@ FUZZ_TARGET(txgraph)
// Remove all remaining transactions, because Refs cannot be destroyed otherwise (this will be // Remove all remaining transactions, because Refs cannot be destroyed otherwise (this will be
// addressed in a follow-up commit). // addressed in a follow-up commit).
for (auto i : sim.graph.Positions()) { for (auto& sim : sims) {
auto ref = sim.GetRef(i); for (auto i : sim.graph.Positions()) {
real->RemoveTransaction(*ref); auto ref = sim.GetRef(i);
real->RemoveTransaction(*ref);
}
} }
} }

812
src/txgraph.cpp
View file

File diff suppressed because it is too large Load diff

101
src/txgraph.h
View file

@ -16,15 +16,18 @@ static constexpr unsigned MAX_CLUSTER_COUNT_LIMIT{64};
/** Data structure to encapsulate fees, sizes, and dependencies for a set of transactions. /** Data structure to encapsulate fees, sizes, and dependencies for a set of transactions.
* *
* The connected components within the transaction graph are called clusters: whenever one * Each TxGraph represents one or two such graphs ("main", and optionally "staging"), to allow for
* working with batches of changes that may still be discarded.
*
* The connected components within each transaction graph are called clusters: whenever one
* transaction is reachable from another, through any sequence of is-parent-of or is-child-of * transaction is reachable from another, through any sequence of is-parent-of or is-child-of
* relations, they belong to the same cluster (so clusters include parents, children, but also * relations, they belong to the same cluster (so clusters include parents, children, but also
* grandparents, siblings, cousins twice removed, ...). * grandparents, siblings, cousins twice removed, ...).
* *
* TxGraph implicitly defines an associated total ordering on its transactions (its linearization) * For each graph, TxGraph implicitly defines an associated total ordering on its transactions
* that respects topology (parents go before their children), aiming for it to be close to the * (its linearization) that respects topology (parents go before their children), aiming for it to
* optimal order those transactions should be mined in if the goal is fee maximization, though this * be close to the optimal order those transactions should be mined in if the goal is fee
* is a best effort only, not a strong guarantee. * maximization, though this is a best effort only, not a strong guarantee.
* *
* For more explanation, see https://delvingbitcoin.org/t/introduction-to-cluster-linearization/1032 * For more explanation, see https://delvingbitcoin.org/t/introduction-to-cluster-linearization/1032
* *
@ -56,11 +59,13 @@ public:
/** Virtual destructor, so inheriting is safe. */ /** Virtual destructor, so inheriting is safe. */
virtual ~TxGraph() = default; virtual ~TxGraph() = default;
/** Construct a new transaction with the specified feerate, and return a Ref to it. In all /** Construct a new transaction with the specified feerate, and return a Ref to it.
* If a staging graph exists, the new transaction is only created there. In all
* further calls, only Refs created by AddTransaction() are allowed to be passed to this * further calls, only Refs created by AddTransaction() are allowed to be passed to this
* TxGraph object (or empty Ref objects). */ * TxGraph object (or empty Ref objects). */
[[nodiscard]] virtual Ref AddTransaction(const FeePerWeight& feerate) noexcept = 0; [[nodiscard]] virtual Ref AddTransaction(const FeePerWeight& feerate) noexcept = 0;
/** Remove the specified transaction. This is a no-op if the transaction was already removed. /** Remove the specified transaction. If a staging graph exists, the removal only happens
* there. This is a no-op if the transaction was already removed.
* *
* TxGraph may internally reorder transaction removals with dependency additions for * TxGraph may internally reorder transaction removals with dependency additions for
* performance reasons. If together with any transaction removal all its descendants, or all * performance reasons. If together with any transaction removal all its descendants, or all
@ -74,42 +79,64 @@ public:
* original order case and the reordered case. * original order case and the reordered case.
*/ */
virtual void RemoveTransaction(const Ref& arg) noexcept = 0; virtual void RemoveTransaction(const Ref& arg) noexcept = 0;
/** Add a dependency between two specified transactions. Parent may not be a descendant of /** Add a dependency between two specified transactions. If a staging graph exists, the
* child already (but may be an ancestor of it already, in which case this is a no-op). If * dependency is only added there. Parent may not be a descendant of child already (but may
* either transaction is already removed, this is a no-op. */ * be an ancestor of it already, in which case this is a no-op). If either transaction is
* already removed, this is a no-op. */
virtual void AddDependency(const Ref& parent, const Ref& child) noexcept = 0; virtual void AddDependency(const Ref& parent, const Ref& child) noexcept = 0;
/** Modify the fee of the specified transaction. If the transaction does not exist (or was /** Modify the fee of the specified transaction, in both the main graph and the staging
* removed), this has no effect. */ * graph if it exists. Wherever the transaction does not exist (or was removed), this has no
* effect. */
virtual void SetTransactionFee(const Ref& arg, int64_t fee) noexcept = 0; virtual void SetTransactionFee(const Ref& arg, int64_t fee) noexcept = 0;
/** Create a staging graph (which cannot exist already). This acts as if a full copy of
* the transaction graph is made, upon which further modifications are made. This copy can
* be inspected, and then either discarded, or the main graph can be replaced by it by
* commiting it. */
virtual void StartStaging() noexcept = 0;
/** Discard the existing active staging graph (which must exist). */
virtual void AbortStaging() noexcept = 0;
/** Replace the main graph with the staging graph (which must exist). */
virtual void CommitStaging() noexcept = 0;
/** Check whether a staging graph exists. */
virtual bool HaveStaging() const noexcept = 0;
/** Determine whether the graph is oversized (contains a connected component of more than the /** Determine whether the graph is oversized (contains a connected component of more than the
* configured maximum cluster count). Some of the functions below are not available * configured maximum cluster count). If main_only is false and a staging graph exists, it is
* queried; otherwise the main graph is queried. Some of the functions below are not available
* for oversized graphs. The mutators above are always available. */ * for oversized graphs. The mutators above are always available. */
virtual bool IsOversized() noexcept = 0; virtual bool IsOversized(bool main_only = false) noexcept = 0;
/** Determine whether arg exists in this graph (i.e., was not removed). This is available even /** Determine whether arg exists in the graph (i.e., was not removed). If main_only is false
* for oversized graphs. */ * and a staging graph exists, it is queried; otherwise the main graph is queried. This is
virtual bool Exists(const Ref& arg) noexcept = 0; * available even for oversized graphs. */
virtual bool Exists(const Ref& arg, bool main_only = false) noexcept = 0;
/** Get the individual transaction feerate of transaction arg. Returns the empty FeePerWeight /** Get the individual transaction feerate of transaction arg. Returns the empty FeePerWeight
* if arg does not exist. This is available even for oversized graphs. */ * if arg does not exist in either main or staging. This is available even for oversized
virtual FeePerWeight GetIndividualFeerate(const Ref& arg) noexcept = 0;
/** Get the feerate of the chunk which transaction arg is in. Returns the empty FeePerWeight if
* arg does not exist. The graph must not be oversized. */
virtual FeePerWeight GetChunkFeerate(const Ref& arg) noexcept = 0;
/** Get pointers to all transactions in the cluster which arg is in. The transactions will be
* returned in graph order. The graph must not be oversized. Returns {} if arg does not exist
* in the graph. */
virtual std::vector<Ref*> GetCluster(const Ref& arg) noexcept = 0;
/** Get pointers to all ancestors of the specified transaction (including the transaction
* itself), in unspecified order. The graph must not be oversized. Returns {} if arg does not
* exist in the graph. */
virtual std::vector<Ref*> GetAncestors(const Ref& arg) noexcept = 0;
/** Get pointers to all descendants of the specified transaction (including the transaction
* itself), in unspecified order. The graph must not be oversized. Returns {} if arg does not
* exist in the graph. */
virtual std::vector<Ref*> GetDescendants(const Ref& arg) noexcept = 0;
/** Get the total number of transactions in the graph. This is available even for oversized
* graphs. */ * graphs. */
virtual GraphIndex GetTransactionCount() noexcept = 0; virtual FeePerWeight GetIndividualFeerate(const Ref& arg) noexcept = 0;
/** Get the feerate of the chunk which transaction arg is in, in the main graph. Returns the
* empty FeePerWeight if arg does not exist in the main graph. The main graph must not be
* oversized. */
virtual FeePerWeight GetMainChunkFeerate(const Ref& arg) noexcept = 0;
/** Get pointers to all transactions in the cluster which arg is in. The transactions are
* returned in graph order. If main_only is false and a staging graph exists, it is queried;
* otherwise the main graph is queried. The queried graph must not be oversized. Returns {} if
* arg does not exist in the queried graph. */
virtual std::vector<Ref*> GetCluster(const Ref& arg, bool main_only = false) noexcept = 0;
/** Get pointers to all ancestors of the specified transaction (including the transaction
* itself), in unspecified order. If main_only is false and a staging graph exists, it is
* queried; otherwise the main graph is queried. The queried graph must not be oversized.
* Returns {} if arg does not exist in the graph. */
virtual std::vector<Ref*> GetAncestors(const Ref& arg, bool main_only = false) noexcept = 0;
/** Get pointers to all descendants of the specified transaction (including the transaction
* itself), in unspecified order. If main_only is false and a staging graph exists, it is
* queried; otherwise the main graph is queried. The queried graph must not be oversized.
* Returns {} if arg does not exist in the graph. */
virtual std::vector<Ref*> GetDescendants(const Ref& arg, bool main_only = false) noexcept = 0;
/** Get the total number of transactions in the graph. If main_only is false and a staging
* graph exists, it is queried; otherwise the main graph is queried. This is available even
* for oversized graphs. */
virtual GraphIndex GetTransactionCount(bool main_only = false) noexcept = 0;
/** Perform an internal consistency check on this object. */ /** Perform an internal consistency check on this object. */
virtual void SanityCheck() const = 0; virtual void SanityCheck() const = 0;
@ -141,7 +168,7 @@ public:
* TxGraph::AddTransaction. */ * TxGraph::AddTransaction. */
Ref() noexcept = default; Ref() noexcept = default;
/** Destroy this Ref. This is only allowed when it is empty, or the transaction it refers /** Destroy this Ref. This is only allowed when it is empty, or the transaction it refers
* to has been removed from the graph. */ * to does not exist in the graph (in main nor staging). */
virtual ~Ref(); virtual ~Ref();
// Support moving a Ref. // Support moving a Ref.
Ref& operator=(Ref&& other) noexcept; Ref& operator=(Ref&& other) noexcept;