From 4f8958d7563ae2d0d359ec1e6885f8cb5e40a5e0 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 19 May 2024 08:03:57 -0400 Subject: [PATCH] clusterlin: add PostLinearize + benchmarks + fuzz tests --- src/bench/cluster_linearize.cpp | 25 ++++ src/cluster_linearize.h | 203 ++++++++++++++++++++++++++++ src/test/fuzz/cluster_linearize.cpp | 163 ++++++++++++++++++++++ 3 files changed, 391 insertions(+) diff --git a/src/bench/cluster_linearize.cpp b/src/bench/cluster_linearize.cpp index 9987d376a5..30c7ecef01 100644 --- a/src/bench/cluster_linearize.cpp +++ b/src/bench/cluster_linearize.cpp @@ -169,6 +169,17 @@ void BenchLinearizeNoItersWorstCaseLIMO(ClusterIndex ntx, benchmark::Bench& benc }); } +template +void BenchPostLinearizeWorstCase(ClusterIndex ntx, benchmark::Bench& bench) +{ + DepGraph depgraph = MakeWideGraph(ntx); + std::vector lin(ntx); + bench.run([&] { + for (ClusterIndex i = 0; i < ntx; ++i) lin[i] = i; + PostLinearize(depgraph, lin); + }); +} + } // namespace static void LinearizePerIter16TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase>(16, bench); } @@ -192,6 +203,13 @@ static void LinearizeNoIters64TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLi static void LinearizeNoIters75TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(75, bench); } static void LinearizeNoIters99TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(99, bench); } +static void PostLinearize16TxWorstCase(benchmark::Bench& bench) { BenchPostLinearizeWorstCase>(16, bench); } +static void PostLinearize32TxWorstCase(benchmark::Bench& bench) { BenchPostLinearizeWorstCase>(32, bench); } +static void PostLinearize48TxWorstCase(benchmark::Bench& bench) { BenchPostLinearizeWorstCase>(48, bench); } +static void PostLinearize64TxWorstCase(benchmark::Bench& bench) { BenchPostLinearizeWorstCase>(64, bench); } +static void PostLinearize75TxWorstCase(benchmark::Bench& bench) { BenchPostLinearizeWorstCase>(75, bench); } +static void PostLinearize99TxWorstCase(benchmark::Bench& bench) { BenchPostLinearizeWorstCase>(99, bench); } + BENCHMARK(LinearizePerIter16TxWorstCase, benchmark::PriorityLevel::HIGH); BENCHMARK(LinearizePerIter32TxWorstCase, benchmark::PriorityLevel::HIGH); BENCHMARK(LinearizePerIter48TxWorstCase, benchmark::PriorityLevel::HIGH); @@ -212,3 +230,10 @@ BENCHMARK(LinearizeNoIters48TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); BENCHMARK(LinearizeNoIters64TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); BENCHMARK(LinearizeNoIters75TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); BENCHMARK(LinearizeNoIters99TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); + +BENCHMARK(PostLinearize16TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(PostLinearize32TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(PostLinearize48TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(PostLinearize64TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(PostLinearize75TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(PostLinearize99TxWorstCase, benchmark::PriorityLevel::HIGH); diff --git a/src/cluster_linearize.h b/src/cluster_linearize.h index b581f01da5..1e02d9fc3b 100644 --- a/src/cluster_linearize.h +++ b/src/cluster_linearize.h @@ -122,6 +122,8 @@ public: auto TxCount() const noexcept { return entries.size(); } /** Get the feerate of a given transaction i. Complexity: O(1). */ const FeeFrac& FeeRate(ClusterIndex i) const noexcept { return entries[i].feerate; } + /** Get the mutable feerate of a given transaction i. Complexity: O(1). */ + FeeFrac& FeeRate(ClusterIndex i) noexcept { return entries[i].feerate; } /** Get the ancestors of a given transaction i. Complexity: O(1). */ const SetType& Ancestors(ClusterIndex i) const noexcept { return entries[i].ancestors; } /** Get the descendants of a given transaction i. Complexity: O(1). */ @@ -782,6 +784,207 @@ std::pair, bool> Linearize(const DepGraph& de return {std::move(linearization), optimal}; } +/** Improve a given linearization. + * + * @param[in] depgraph Dependency graph of the cluster being linearized. + * @param[in,out] linearization On input, an existing linearization for depgraph. On output, a + * potentially better linearization for the same graph. + * + * Postlinearization guarantees: + * - The resulting chunks are connected. + * - If the input has a tree shape (either all transactions have at most one child, or all + * transactions have at most one parent), the result is optimal. + * - Given a linearization L1 and a leaf transaction T in it. Let L2 be L1 with T moved to the end, + * optionally with its fee increased. Let L3 be the postlinearization of L2. L3 will be at least + * as good as L1. This means that replacing transactions with same-size higher-fee transactions + * will not worsen linearizations through a "drop conflicts, append new transactions, + * postlinearize" process. + */ +template +void PostLinearize(const DepGraph& depgraph, Span linearization) +{ + // This algorithm performs a number of passes (currently 2); the even ones operate from back to + // front, the odd ones from front to back. Each results in an equal-or-better linearization + // than the one started from. + // - One pass in either direction guarantees that the resulting chunks are connected. + // - Each direction corresponds to one shape of tree being linearized optimally (forward passes + // guarantee this for graphs where each transaction has at most one child; backward passes + // guarantee this for graphs where each transaction has at most one parent). + // - Starting with a backward pass guarantees the moved-tree property. + // + // During an odd (forward) pass, the high-level operation is: + // - Start with an empty list of groups L=[]. + // - For every transaction i in the old linearization, from front to back: + // - Append a new group C=[i], containing just i, to the back of L. + // - While L has at least one group before C, and the group immediately before C has feerate + // lower than C: + // - If C depends on P: + // - Merge P into C, making C the concatenation of P+C, continuing with the combined C. + // - Otherwise: + // - Swap P with C, continuing with the now-moved C. + // - The output linearization is the concatenation of the groups in L. + // + // During even (backward) passes, i iterates from the back to the front of the existing + // linearization, and new groups are prepended instead of appended to the list L. To enable + // more code reuse, both passes append groups, but during even passes the meanings of + // parent/child, and of high/low feerate are reversed, and the final concatenation is reversed + // on output. + // + // In the implementation below, the groups are represented by singly-linked lists (pointing + // from the back to the front), which are themselves organized in a singly-linked circular + // list (each group pointing to its predecessor, with a special sentinel group at the front + // that points back to the last group). + // + // Information about transaction t is stored in entries[t + 1], while the sentinel is in + // entries[0]. + + /** Index of the sentinel in the entries array below. */ + static constexpr ClusterIndex SENTINEL{0}; + /** Indicator that a group has no previous transaction. */ + static constexpr ClusterIndex NO_PREV_TX{0}; + + + /** Data structure per transaction entry. */ + struct TxEntry + { + /** The index of the previous transaction in this group; NO_PREV_TX if this is the first + * entry of a group. */ + ClusterIndex prev_tx; + + // The fields below are only used for transactions that are the last one in a group + // (referred to as tail transactions below). + + /** Index of the first transaction in this group, possibly itself. */ + ClusterIndex first_tx; + /** Index of the last transaction in the previous group. The first group (the sentinel) + * points back to the last group here, making it a singly-linked circular list. */ + ClusterIndex prev_group; + /** All transactions in the group. Empty for the sentinel. */ + SetType group; + /** All dependencies of the group (descendants in even passes; ancestors in odd ones). */ + SetType deps; + /** The combined fee/size of transactions in the group. Fee is negated in even passes. */ + FeeFrac feerate; + }; + + // As an example, consider the state corresponding to the linearization [1,0,3,2], with + // groups [1,0,3] and [2], in an odd pass. The linked lists would be: + // + // +-----+ + // 0<-P-- | 0 S | ---\ Legend: + // +-----+ | + // ^ | - digit in box: entries index + // /--------------F---------+ G | (note: one more than tx value) + // v \ | | - S: sentinel group + // +-----+ +-----+ +-----+ | (empty feerate) + // 0<-P-- | 2 | <--P-- | 1 | <--P-- | 4 T | | - T: tail transaction, contains + // +-----+ +-----+ +-----+ | fields beyond prev_tv. + // ^ | - P: prev_tx reference + // G G - F: first_tx reference + // | | - G: prev_group reference + // +-----+ | + // 0<-P-- | 3 T | <--/ + // +-----+ + // ^ | + // \-F-/ + // + // During an even pass, the diagram above would correspond to linearization [2,3,0,1], with + // groups [2] and [3,0,1]. + + std::vector entries(linearization.size() + 1); + + // Perform two passes over the linearization. + for (int pass = 0; pass < 2; ++pass) { + int rev = !(pass & 1); + // Construct a sentinel group, identifying the start of the list. + entries[SENTINEL].prev_group = SENTINEL; + Assume(entries[SENTINEL].feerate.IsEmpty()); + + // Iterate over all elements in the existing linearization. + for (ClusterIndex i = 0; i < linearization.size(); ++i) { + // Even passes are from back to front; odd passes from front to back. + ClusterIndex idx = linearization[rev ? linearization.size() - 1 - i : i]; + // Construct a new group containing just idx. In even passes, the meaning of + // parent/child and high/low feerate are swapped. + ClusterIndex cur_group = idx + 1; + entries[cur_group].group = SetType::Singleton(idx); + entries[cur_group].deps = rev ? depgraph.Descendants(idx): depgraph.Ancestors(idx); + entries[cur_group].feerate = depgraph.FeeRate(idx); + if (rev) entries[cur_group].feerate.fee = -entries[cur_group].feerate.fee; + entries[cur_group].prev_tx = NO_PREV_TX; // No previous transaction in group. + entries[cur_group].first_tx = cur_group; // Transaction itself is first of group. + // Insert the new group at the back of the groups linked list. + entries[cur_group].prev_group = entries[SENTINEL].prev_group; + entries[SENTINEL].prev_group = cur_group; + + // Start merge/swap cycle. + ClusterIndex next_group = SENTINEL; // We inserted at the end, so next group is sentinel. + ClusterIndex prev_group = entries[cur_group].prev_group; + // Continue as long as the current group has higher feerate than the previous one. + while (entries[cur_group].feerate >> entries[prev_group].feerate) { + // prev_group/cur_group/next_group refer to (the last transactions of) 3 + // consecutive entries in groups list. + Assume(cur_group == entries[next_group].prev_group); + Assume(prev_group == entries[cur_group].prev_group); + // The sentinel has empty feerate, which is neither higher or lower than other + // feerates. Thus, the while loop we are in here guarantees that cur_group and + // prev_group are not the sentinel. + Assume(cur_group != SENTINEL); + Assume(prev_group != SENTINEL); + if (entries[cur_group].deps.Overlaps(entries[prev_group].group)) { + // There is a dependency between cur_group and prev_group; merge prev_group + // into cur_group. The group/deps/feerate fields of prev_group remain unchanged + // but become unused. + entries[cur_group].group |= entries[prev_group].group; + entries[cur_group].deps |= entries[prev_group].deps; + entries[cur_group].feerate += entries[prev_group].feerate; + // Make the first of the current group point to the tail of the previous group. + entries[entries[cur_group].first_tx].prev_tx = prev_group; + // The first of the previous group becomes the first of the newly-merged group. + entries[cur_group].first_tx = entries[prev_group].first_tx; + // The previous group becomes whatever group was before the former one. + prev_group = entries[prev_group].prev_group; + entries[cur_group].prev_group = prev_group; + } else { + // There is no dependency between cur_group and prev_group; swap them. + ClusterIndex preprev_group = entries[prev_group].prev_group; + // If PP, P, C, N were the old preprev, prev, cur, next groups, then the new + // layout becomes [PP, C, P, N]. Update prev_groups to reflect that order. + entries[next_group].prev_group = prev_group; + entries[prev_group].prev_group = cur_group; + entries[cur_group].prev_group = preprev_group; + // The current group remains the same, but the groups before/after it have + // changed. + next_group = prev_group; + prev_group = preprev_group; + } + } + } + + // Convert the entries back to linearization (overwriting the existing one). + ClusterIndex cur_group = entries[0].prev_group; + ClusterIndex done = 0; + while (cur_group != SENTINEL) { + ClusterIndex cur_tx = cur_group; + // Traverse the transactions of cur_group (from back to front), and write them in the + // same order during odd passes, and reversed (front to back) in even passes. + if (rev) { + do { + *(linearization.begin() + (done++)) = cur_tx - 1; + cur_tx = entries[cur_tx].prev_tx; + } while (cur_tx != NO_PREV_TX); + } else { + do { + *(linearization.end() - (++done)) = cur_tx - 1; + cur_tx = entries[cur_tx].prev_tx; + } while (cur_tx != NO_PREV_TX); + } + cur_group = entries[cur_group].prev_group; + } + Assume(done == linearization.size()); + } +} + } // namespace cluster_linearize #endif // BITCOIN_CLUSTER_LINEARIZE_H diff --git a/src/test/fuzz/cluster_linearize.cpp b/src/test/fuzz/cluster_linearize.cpp index 1d16432c9a..2412db5c1b 100644 --- a/src/test/fuzz/cluster_linearize.cpp +++ b/src/test/fuzz/cluster_linearize.cpp @@ -766,3 +766,166 @@ FUZZ_TARGET(clusterlin_linearize) } } } + +FUZZ_TARGET(clusterlin_postlinearize) +{ + // Verify expected properties of PostLinearize() on arbitrary linearizations. + + // Retrieve a depgraph from the fuzz input. + SpanReader reader(buffer); + DepGraph depgraph; + try { + reader >> Using(depgraph); + } catch (const std::ios_base::failure&) {} + + // Retrieve a linearization from the fuzz input. + std::vector linearization; + linearization = ReadLinearization(depgraph, reader); + SanityCheck(depgraph, linearization); + + // Produce a post-processed version. + auto post_linearization = linearization; + PostLinearize(depgraph, post_linearization); + SanityCheck(depgraph, post_linearization); + + // Compare diagrams: post-linearization cannot worsen anywhere. + auto chunking = ChunkLinearization(depgraph, linearization); + auto post_chunking = ChunkLinearization(depgraph, post_linearization); + auto cmp = CompareChunks(post_chunking, chunking); + assert(cmp >= 0); + + // Run again, things can keep improving (and never get worse) + auto post_post_linearization = post_linearization; + PostLinearize(depgraph, post_post_linearization); + SanityCheck(depgraph, post_post_linearization); + auto post_post_chunking = ChunkLinearization(depgraph, post_post_linearization); + cmp = CompareChunks(post_post_chunking, post_chunking); + assert(cmp >= 0); + + // The chunks that come out of postlinearizing are always connected. + LinearizationChunking linchunking(depgraph, post_linearization); + while (linchunking.NumChunksLeft()) { + assert(depgraph.IsConnected(linchunking.GetChunk(0).transactions)); + linchunking.MarkDone(linchunking.GetChunk(0).transactions); + } +} + +FUZZ_TARGET(clusterlin_postlinearize_tree) +{ + // Verify expected properties of PostLinearize() on linearizations of graphs that form either + // an upright or reverse tree structure. + + // Construct a direction, RNG seed, and an arbitrary graph from the fuzz input. + SpanReader reader(buffer); + uint64_t rng_seed{0}; + DepGraph depgraph_gen; + uint8_t direction{0}; + try { + reader >> direction >> rng_seed >> Using(depgraph_gen); + } catch (const std::ios_base::failure&) {} + + // Now construct a new graph, copying the nodes, but leaving only the first parent (even + // direction) or the first child (odd direction). + DepGraph depgraph_tree; + for (ClusterIndex i = 0; i < depgraph_gen.TxCount(); ++i) { + depgraph_tree.AddTransaction(depgraph_gen.FeeRate(i)); + } + if (direction & 1) { + for (ClusterIndex i = 0; i < depgraph_gen.TxCount(); ++i) { + auto children = depgraph_gen.Descendants(i) - TestBitSet::Singleton(i); + // Remove descendants that are children of other descendants. + for (auto j : children) { + if (!children[j]) continue; + children -= depgraph_gen.Descendants(j); + children.Set(j); + } + if (children.Any()) depgraph_tree.AddDependency(i, children.First()); + } + } else { + for (ClusterIndex i = 0; i < depgraph_gen.TxCount(); ++i) { + auto parents = depgraph_gen.Ancestors(i) - TestBitSet::Singleton(i); + // Remove ancestors that are parents of other ancestors. + for (auto j : parents) { + if (!parents[j]) continue; + parents -= depgraph_gen.Ancestors(j); + parents.Set(j); + } + if (parents.Any()) depgraph_tree.AddDependency(parents.First(), i); + } + } + + // Retrieve a linearization from the fuzz input. + std::vector linearization; + linearization = ReadLinearization(depgraph_tree, reader); + SanityCheck(depgraph_tree, linearization); + + // Produce a postlinearized version. + auto post_linearization = linearization; + PostLinearize(depgraph_tree, post_linearization); + SanityCheck(depgraph_tree, post_linearization); + + // Compare diagrams. + auto chunking = ChunkLinearization(depgraph_tree, linearization); + auto post_chunking = ChunkLinearization(depgraph_tree, post_linearization); + auto cmp = CompareChunks(post_chunking, chunking); + assert(cmp >= 0); + + // Verify that post-linearizing again does not change the diagram. The result must be identical + // as post_linearization ought to be optimal already with a tree-structured graph. + auto post_post_linearization = post_linearization; + PostLinearize(depgraph_tree, post_linearization); + SanityCheck(depgraph_tree, post_linearization); + auto post_post_chunking = ChunkLinearization(depgraph_tree, post_post_linearization); + auto cmp_post = CompareChunks(post_post_chunking, post_chunking); + assert(cmp_post == 0); + + // Try to find an even better linearization directly. This must not change the diagram for the + // same reason. + auto [opt_linearization, _optimal] = Linearize(depgraph_tree, 100000, rng_seed, post_linearization); + auto opt_chunking = ChunkLinearization(depgraph_tree, opt_linearization); + auto cmp_opt = CompareChunks(opt_chunking, post_chunking); + assert(cmp_opt == 0); +} + +FUZZ_TARGET(clusterlin_postlinearize_moved_leaf) +{ + // Verify that taking an existing linearization, and moving a leaf to the back, potentially + // increasing its fee, and then post-linearizing, results in something as good as the + // original. This guarantees that in an RBF that replaces a transaction with one of the same + // size but higher fee, applying the "remove conflicts, append new transaction, postlinearize" + // process will never worsen linearization quality. + + // Construct an arbitrary graph and a fee from the fuzz input. + SpanReader reader(buffer); + DepGraph depgraph; + int32_t fee_inc{0}; + try { + uint64_t fee_inc_code; + reader >> Using(depgraph) >> VARINT(fee_inc_code); + fee_inc = fee_inc_code & 0x3ffff; + } catch (const std::ios_base::failure&) {} + if (depgraph.TxCount() == 0) return; + + // Retrieve two linearizations from the fuzz input. + auto lin = ReadLinearization(depgraph, reader); + auto lin_leaf = ReadLinearization(depgraph, reader); + + // Construct a linearization identical to lin, but with the tail end of lin_leaf moved to the + // back. + std::vector lin_moved; + for (auto i : lin) { + if (i != lin_leaf.back()) lin_moved.push_back(i); + } + lin_moved.push_back(lin_leaf.back()); + + // Postlinearize lin_moved. + PostLinearize(depgraph, lin_moved); + SanityCheck(depgraph, lin_moved); + + // Compare diagrams (applying the fee delta after computing the old one). + auto old_chunking = ChunkLinearization(depgraph, lin); + depgraph.FeeRate(lin_leaf.back()).fee += fee_inc; + auto new_chunking = ChunkLinearization(depgraph, lin_moved); + auto cmp = CompareChunks(new_chunking, old_chunking); + assert(cmp >= 0); +}