txgraph: add work limit to DoWork(), try optimal (feature)

This adds an `iters` parameter to DoWork(), which controls how much work it is
allowed to do right now.

Additionally, DoWork() won't stop at just getting everything ACCEPTABLE, but if
there is work budget left, will also attempt to get every cluster linearized
optimally.

src/test/fuzz/txgraph.cpp

@@ -58,6 +58,8 @@ struct SimTxGraph
     SetType modified;
     /** The configured maximum total size of transactions per cluster. */
     uint64_t max_cluster_size;
+    /** Whether the corresponding real graph is known to be optimally linearized. */
+    bool real_is_optimal{false};
 
     /** Construct a new SimTxGraph with the specified maximum cluster count. */
     explicit SimTxGraph(DepGraphIndex max_cluster, uint64_t max_size) :
@@ -129,6 +131,7 @@ struct SimTxGraph
     {
         assert(graph.TxCount() < MAX_TRANSACTIONS);
         auto simpos = graph.AddTransaction(feerate);
+        real_is_optimal = false;
         MakeModified(simpos);
         assert(graph.Positions()[simpos]);
         simmap[simpos] = std::make_shared<TxGraph::Ref>();
@@ -148,6 +151,7 @@ struct SimTxGraph
         if (chl_pos == MISSING) return;
         graph.AddDependencies(SetType::Singleton(par_pos), chl_pos);
         MakeModified(par_pos);
+        real_is_optimal = false;
         // This may invalidate our cached oversized value.
         if (oversized.has_value() && !*oversized) oversized = std::nullopt;
     }
@@ -158,6 +162,7 @@ struct SimTxGraph
         auto pos = Find(ref);
         if (pos == MISSING) return;
         // No need to invoke MakeModified, because this equally affects main and staging.
+        real_is_optimal = false;
         graph.FeeRate(pos).fee = fee;
     }
 
@@ -167,6 +172,7 @@ struct SimTxGraph
         auto pos = Find(ref);
         if (pos == MISSING) return;
         MakeModified(pos);
+        real_is_optimal = false;
         graph.RemoveTransactions(SetType::Singleton(pos));
         simrevmap.erase(simmap[pos].get());
         // Retain the TxGraph::Ref corresponding to this position, so the Ref destruction isn't
@@ -193,6 +199,7 @@ struct SimTxGraph
         } else {
             MakeModified(pos);
             graph.RemoveTransactions(SetType::Singleton(pos));
+            real_is_optimal = false;
             simrevmap.erase(simmap[pos].get());
             simmap[pos].reset();
             // This may invalidate our cached oversized value.
@@ -430,6 +437,7 @@ FUZZ_TARGET(txgraph)
                     if (top_sim.graph.Ancestors(pos_par)[pos_chl]) break;
                 }
                 top_sim.AddDependency(par, chl);
+                top_sim.real_is_optimal = false;
                 real->AddDependency(*par, *chl);
                 break;
             } else if ((block_builders.empty() || sims.size() > 1) && top_sim.removed.size() < 100 && command-- == 0) {
@@ -684,7 +692,10 @@ FUZZ_TARGET(txgraph)
                 break;
             } else if (command-- == 0) {
                 // DoWork.
-                real->DoWork();
+                uint64_t iters = provider.ConsumeIntegralInRange<uint64_t>(0, alt ? 10000 : 255);
+                if (real->DoWork(iters)) {
+                    for (auto& sim : sims) sim.real_is_optimal = true;
+                }
                 break;
             } else if (sims.size() == 2 && !sims[0].IsOversized() && !sims[1].IsOversized() && command-- == 0) {
                 // GetMainStagingDiagrams()
@@ -849,6 +860,16 @@ FUZZ_TARGET(txgraph)
         }
         assert(todo.None());
 
+        // If the real graph claims to be optimal (the last DoWork() call returned true), verify
+        // that calling Linearize on it does not improve it further.
+        if (sims[0].real_is_optimal) {
+            auto real_diagram = ChunkLinearization(sims[0].graph, vec1);
+            auto [sim_lin, _optimal, _cost] = Linearize(sims[0].graph, 300000, rng.rand64(), vec1);
+            auto sim_diagram = ChunkLinearization(sims[0].graph, sim_lin);
+            auto cmp = CompareChunks(real_diagram, sim_diagram);
+            assert(cmp == 0);
+        }
+
         // For every transaction in the total ordering, find a random one before it and after it,
         // and compare their chunk feerates, which must be consistent with the ordering.
         for (size_t pos = 0; pos < vec1.size(); ++pos) {

src/txgraph.cpp

@@ -181,8 +181,9 @@ public:
     void Merge(TxGraphImpl& graph, Cluster& cluster) noexcept;
     /** Given a span of (parent, child) pairs that all belong to this Cluster, apply them. */
     void ApplyDependencies(TxGraphImpl& graph, std::span<std::pair<GraphIndex, GraphIndex>> to_apply) noexcept;
-    /** Improve the linearization of this Cluster. Returns how much work was performed. */
+    /** Improve the linearization of this Cluster. Returns how much work was performed and whether
-    uint64_t Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept;
+     *  the Cluster is now optimal. */
+    std::pair<uint64_t, bool> Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept;
     /** For every chunk in the cluster, append its FeeFrac to ret. */
     void AppendChunkFeerates(std::vector<FeeFrac>& ret) const noexcept;
     /** Add a TrimTxData entry for every transaction in the Cluster to ret. Implicit dependencies
@@ -577,7 +578,7 @@ public:
     void AddDependency(const Ref& parent, const Ref& child) noexcept final;
     void SetTransactionFee(const Ref&, int64_t fee) noexcept final;
 
-    void DoWork() noexcept final;
+    bool DoWork(uint64_t iters) noexcept final;
 
     void StartStaging() noexcept final;
     void CommitStaging() noexcept final;
@@ -1647,12 +1648,12 @@ void TxGraphImpl::ApplyDependencies(int level) noexcept
     clusterset.m_group_data = GroupData{};
 }
 
-uint64_t Cluster::Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept
+std::pair<uint64_t, bool> Cluster::Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept
 {
     // We can only relinearize Clusters that do not need splitting.
     Assume(!NeedsSplitting());
     // No work is required for Clusters which are already optimally linearized.
-    if (IsOptimal()) return 0;
+    if (IsOptimal()) return {0, true};
     // Invoke the actual linearization algorithm (passing in the existing one).
     uint64_t rng_seed = graph.m_rng.rand64();
     auto [linearization, optimal, cost] = Linearize(m_depgraph, max_iters, rng_seed, m_linearization);
@@ -1666,7 +1667,7 @@ uint64_t Cluster::Relinearize(TxGraphImpl& graph, uint64_t max_iters) noexcept
     graph.SetClusterQuality(m_level, m_quality, m_setindex, new_quality);
     // Update the Entry objects.
     Updated(graph);
-    return cost;
+    return {cost, optimal};
 }
 
 uint64_t TxGraphImpl::MakeAcceptable(Cluster& cluster) noexcept
@@ -1674,7 +1675,7 @@ uint64_t TxGraphImpl::MakeAcceptable(Cluster& cluster) noexcept
     uint64_t cost{0};
     // Relinearize the Cluster if needed.
     if (!cluster.NeedsSplitting() && !cluster.IsAcceptable() && !cluster.IsOversized()) {
-        cost += cluster.Relinearize(*this, m_acceptable_iters);
+        cost += cluster.Relinearize(*this, m_acceptable_iters).first;
     }
     return cost;
 }
@@ -2487,13 +2488,37 @@ void TxGraphImpl::SanityCheck() const
     assert(actual_chunkindex == expected_chunkindex);
 }
 
-void TxGraphImpl::DoWork() noexcept
+bool TxGraphImpl::DoWork(uint64_t iters) noexcept
 {
-    for (int level = 0; level <= GetTopLevel(); ++level) {
+    uint64_t iters_done{0};
-        if (level > 0 || m_main_chunkindex_observers == 0) {
+    // Relinearize everything to acceptable level first.
-            MakeAllAcceptable(level);
+    for (int level = GetTopLevel(); level >= 0; --level) {
+        if (level == 0 && m_main_chunkindex_observers != 0) continue;
+        ApplyDependencies(level);
+        auto& clusterset = GetClusterSet(level);
+        if (clusterset.m_oversized == true) continue;
+        auto& queue = clusterset.m_clusters[int(QualityLevel::NEEDS_RELINEARIZE)];
+        while (!queue.empty()) {
+            if (iters_done + m_acceptable_iters >= iters) return false;
+            iters_done += MakeAcceptable(*queue.back().get());
         }
     }
+    // If we have budget for more work left, get things optimal.
+    for (int level = GetTopLevel(); level >= 0; --level) {
+        if (level == 0 && m_main_chunkindex_observers != 0) continue;
+        auto& clusterset = GetClusterSet(level);
+        if (clusterset.m_oversized == true) continue;
+        auto& queue = clusterset.m_clusters[int(QualityLevel::ACCEPTABLE)];
+        while (!queue.empty()) {
+            // Randomize the order in which we process, so that if the first cluster somehow needs
+            // more work than what iters allows, we don't keep spending it on the same one.
+            auto pos = m_rng.randrange<size_t>(queue.size());
+            auto [cost, optimal] = queue[pos].get()->Relinearize(*this, iters - iters_done);
+            iters_done += cost;
+            if (!optimal) return false;
+        }
+    }
+    return true;
 }
 
 void BlockBuilderImpl::Next() noexcept

src/txgraph.h

@@ -94,9 +94,10 @@ public:
     virtual void SetTransactionFee(const Ref& arg, int64_t fee) noexcept = 0;
 
     /** TxGraph is internally lazy, and will not compute many things until they are needed.
-     *  Calling DoWork will compute everything now, so that future operations are fast. This can be
+     *  Calling DoWork will perform some work now (controlled by iters) so that future operations
-     *  invoked while oversized. */
+     *  are fast, if there is any. Returns whether all work is done. This can be invoked while
-    virtual void DoWork() noexcept = 0;
+     *  oversized. */
+    virtual bool DoWork(uint64_t iters) noexcept = 0;
 
     /** Create a staging graph (which cannot exist already). This acts as if a full copy of
      *  the transaction graph is made, upon which further modifications are made. This copy can