txgraph: (feature) introduce Evictor interface

Similar to the BlockBuilder interface, this lets one iterate the set of
chunks in the entire graph. The iteration happens from low to high chunk
feerate however, does not permit skipping chunks, but does permit destroying
Refs of the chunks that are being iterated over.

src/test/fuzz/txgraph.cpp

@@ -621,6 +621,53 @@ FUZZ_TARGET(txgraph)
                     }
                 }
                 break;
+            } else if (/*sims.size() == 1 &&*/ !main_sim.IsOversized() && command-- == 0) {
+                // GetEvictor.
+                auto num_to_evict = provider.ConsumeIntegralInRange<int32_t>(0, main_sim.GetTransactionCount());
+                auto evictor = real->GetEvictor();
+                SimTxGraph::SetType done;
+                FeeFrac prev_chunk_feerate;
+                while (*evictor && num_to_evict >= 0) {
+                    // Chunk feerates must be monotonously increasing.
+                    if (!prev_chunk_feerate.IsEmpty()) {
+                        assert(FeeRateCompare(evictor->GetCurrentChunkFeerate(), prev_chunk_feerate) >= 0);
+                    }
+                    prev_chunk_feerate = evictor->GetCurrentChunkFeerate();
+                    FeeFrac sum_feerate;
+                    for (TxGraph::Ref* ref : evictor->GetCurrentChunk()) {
+                        // Each transaction in the chunk must exist in the main graph.
+                        auto simpos = main_sim.Find(*ref);
+                        assert(simpos != SimTxGraph::MISSING);
+                        // Verify the claimed chunk feerate.
+                        sum_feerate += main_sim.graph.FeeRate(simpos);
+                        // Make sure the chunk contains no duplicate transactions.
+                        assert(!done[simpos]);
+                        done.Set(simpos);
+                        // The concatenation of all reported transaction, in order, must be
+                        // anti-topologically valid (all children before parents).
+                        assert(main_sim.graph.Descendants(simpos).IsSubsetOf(done));
+                        if (num_to_evict > 0) {
+                            // Before destroying Ref, also remove any descendants it may have in
+                            // staging, so that dependencies are consistent.
+                            if (sims.size() == 2) {
+                                auto stage_simpos = top_sim.Find(*ref);
+                                if (stage_simpos != SimTxGraph::MISSING) {
+                                    for (auto desc : top_sim.graph.Descendants(stage_simpos)) {
+                                        auto& desc_ref = top_sim.GetRef(desc);
+                                        top_sim.RemoveTransaction(desc_ref);
+                                        real->RemoveTransaction(desc_ref);
+                                    }
+                                }
+                            }
+                            // Destroy the Ref for both sims.
+                            for (auto& sim : sims) sim.DestroyTransaction(*ref, true);
+                            --num_to_evict;
+                        }
+                    }
+                    assert(sum_feerate == evictor->GetCurrentChunkFeerate());
+                    evictor->Next();
+                }
+                break;
             }
         }
     }
@@ -695,6 +742,30 @@ FUZZ_TARGET(txgraph)
             builder->Include();
         }
         assert(vec_builder == vec1);
+        builder.reset();
+
+        // The reverse order should be obtained through an Evictor, if nothing is destroyed.
+        auto evictor = real->GetEvictor();
+        std::vector<SimTxGraph::Pos> vec_evictor;
+        while (*evictor) {
+            FeeFrac sum;
+            for (TxGraph::Ref* ref : evictor->GetCurrentChunk()) {
+                // The reported chunk feerate must match the chunk feerate obtained by asking
+                // it for each of the chunk's transactions individually.
+                assert(real->GetMainChunkFeerate(*ref) == evictor->GetCurrentChunkFeerate());
+                // Verify the chunk feerate matches the sum of the reported individual feerates.
+                sum += real->GetIndividualFeerate(*ref);
+                // Chunks must contain transactions that exist in the graph.
+                auto simpos = sims[0].Find(*ref);
+                assert(simpos != SimTxGraph::MISSING);
+                vec_evictor.push_back(simpos);
+            }
+            assert(sum == evictor->GetCurrentChunkFeerate());
+            evictor->Next();
+        }
+        std::reverse(vec_evictor.begin(), vec_evictor.end());
+        assert(vec_evictor == vec1);
+        evictor.reset();
 
         // Check that the implied ordering gives rise to a combined diagram that matches the
         // diagram constructed from the individual cluster linearization chunkings.

src/txgraph.cpp

@@ -167,6 +167,7 @@ class TxGraphImpl final : public TxGraph
 {
     friend class Cluster;
     friend class BlockBuilderImpl;
+    friend class EvictorImpl;
 private:
     /** Internal RNG. */
     FastRandomContext m_rng;
@@ -254,7 +255,7 @@ private:
 
     /** Index of ChunkData objects. */
     ChunkIndex m_chunkindex;
-    /** Number of index-observing objects in existence (BlockBuilderImpl). */
+    /** Number of index-observing objects in existence (BlockBuilderImpl, EvictorImpl). */
     size_t m_chunkindex_observers{0};
 
     /** A Locator that describes whether, where, and in which Cluster an Entry appears.
@@ -362,7 +363,6 @@ public:
     void UnlinkRef(GraphIndex idx) noexcept final
     {
         auto& entry = m_entries[idx];
-        Assume(m_chunkindex_observers == 0);
         Assume(entry.m_ref != nullptr);
         entry.m_ref = nullptr;
         if (!entry.IsWiped()) {
@@ -418,6 +418,7 @@ public:
     std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>> GetMainStagingDiagrams() noexcept final;
 
     std::unique_ptr<BlockBuilder> GetBlockBuilder() noexcept final;
+    std::unique_ptr<Evictor> GetEvictor() noexcept final;
 
     void SanityCheck() const final;
 };
@@ -453,6 +454,27 @@ public:
     void Skip() noexcept final;
 };
 
+/** Implementation of the TxGraph::Evictor interface. */
+class EvictorImpl final : public TxGraph::Evictor
+{
+    /** Which TxGraphImpl this object is doing eviction. It will have its m_chunkindex_observers
+     *  incremented as long as this EvictorImpl exists. */
+    TxGraphImpl* const m_graph;
+    /** Vector for actual storage pointed to by TxGraph::Evictor::m_current_chunk. */
+    std::vector<TxGraph::Ref*> m_chunkdata;
+    /** Iterator to the next chunk (after the current one) in the chunk index. rend() if nothing
+     *  further remains. */
+    TxGraphImpl::ChunkIndex::const_reverse_iterator m_next_iter;
+
+public:
+    /** Construct a new EvictorImpl for the provided graph. */
+    EvictorImpl(TxGraphImpl& graph) noexcept;
+
+    // Implement the public interface.
+    ~EvictorImpl() final;
+    void Next() noexcept final;
+};
+
 void TxGraphImpl::ClearLocator(int level, GraphIndex idx) noexcept
 {
     auto& entry = m_entries[idx];
@@ -2042,6 +2064,55 @@ std::unique_ptr<TxGraph::BlockBuilder> TxGraphImpl::GetBlockBuilder() noexcept
     return std::make_unique<BlockBuilderImpl>(*this);
 }
 
+void EvictorImpl::Next() noexcept
+{
+    while (m_next_iter != m_graph->m_chunkindex.rend()) {
+        // Find the cluster pointed to by m_next_iter (and advance it).
+        const auto& chunk_data = *(m_next_iter++);
+        const auto& chunk_end_entry = m_graph->m_entries[chunk_data.m_graph_index];
+        Cluster* cluster = chunk_end_entry.m_locator[0].cluster;
+        // Populate m_current_chunk.
+        m_chunkdata.resize(chunk_data.m_chunk_count);
+        auto start_pos = chunk_end_entry.m_main_lin_index + 1 - chunk_data.m_chunk_count;
+        cluster->GetClusterRefs(*m_graph, m_chunkdata, start_pos);
+        m_current_chunk.emplace(m_chunkdata, chunk_end_entry.m_main_chunk_feerate);
+        // GetClusterRefs emits in topological order; Evictor interface expects children before
+        // parents, so reverse.
+        std::reverse(m_chunkdata.begin(), m_chunkdata.end());
+        return;
+    }
+    // We reached the end of m_chunkindex.
+    m_current_chunk = std::nullopt;
+}
+
+EvictorImpl::EvictorImpl(TxGraphImpl& graph) noexcept : m_graph(&graph)
+{
+    // Make sure all clusters in main are up to date, and acceptable.
+    m_graph->SplitAll(0);
+    if (m_graph->m_clustersets.size() == 1) m_graph->ApplyDependencies();
+    m_graph->MakeAllAcceptable(0);
+    // The main graph cannot be oversized, as that implies unappliable dependencies.
+    Assume(!m_graph->m_clustersets[0].m_oversized);
+    // Remember that this object is observing the graph's index, so that we can detect concurrent
+    // modifications.
+    ++m_graph->m_chunkindex_observers;
+    // Find the first chunk.
+    m_next_iter = m_graph->m_chunkindex.rbegin();
+    Next();
+}
+
+EvictorImpl::~EvictorImpl()
+{
+    Assume(m_graph->m_chunkindex_observers > 0);
+    // Permit modifications to the main graph again after destroying the BlockBuilderImpl.
+    --m_graph->m_chunkindex_observers;
+}
+
+std::unique_ptr<TxGraph::Evictor> TxGraphImpl::GetEvictor() noexcept
+{
+    return std::make_unique<EvictorImpl>(*this);
+}
+
 } // namespace
 
 TxGraph::Ref::~Ref()

src/txgraph.h

@@ -56,29 +56,46 @@ public:
         Ref(const Ref&) = delete;
     };
 
-    /** Interface returned by GetBlockBuilder. */
-    class BlockBuilder
+    /** Base class for BlockBuilder and Evictor. */
+    class ChunkIterator
     {
     protected:
         /** The next chunk, in topological order plus feerate, or std::nullopt if done. */
         std::optional<std::pair<std::span<Ref*>, FeeFrac>> m_current_chunk;
         /** Make constructor non-public (use TxGraph::GetBlockBuilder()). */
-        BlockBuilder() noexcept = default;
+        ChunkIterator() noexcept = default;
     public:
         /** Support safe inheritance. */
-        virtual ~BlockBuilder() = default;
-        /** Determine whether there are more transactions to be included. */
+        virtual ~ChunkIterator() = default;
+        /** Determine whether there are more transactions to be processed. */
         explicit operator bool() noexcept { return m_current_chunk.has_value(); }
         /** Get the chunk that is currently suggested to be included. */
         const std::span<Ref*>& GetCurrentChunk() noexcept { return m_current_chunk->first; }
         /** Get the feerate of the currently suggested chunk. */
         const FeeFrac& GetCurrentChunkFeerate() noexcept { return m_current_chunk->second; }
+    };
+
+    /** Interface returned by GetBlockBuilder. */
+    class BlockBuilder : public ChunkIterator
+    {
+    public:
         /** Mark the current chunk as included, and progress to the next one. */
         virtual void Include() noexcept = 0;
         /** Mark the current chunk as skipped, and progress to the next one. */
         virtual void Skip() noexcept = 0;
     };
 
+    /** Interface returned by GetEvictor. */
+    class Evictor : public ChunkIterator
+    {
+    public:
+        /** Progress to the next chunk. It is allowed to destroy the Ref objects pointed to by
+         *  GetCurrentChunk before calling Next(), but not other modifications to the main graph
+         *  are allowed while the Evictor exists. Children will always be reported before parents.
+         */
+        virtual void Next() noexcept = 0;
+    };
+
 protected:
     // Allow TxGraph::Ref to call UpdateRef and UnlinkRef.
     friend class TxGraph::Ref;
@@ -186,6 +203,10 @@ public:
     /** Construct a block builder, drawing from the main graph, which cannot be oversized. While
      *  the returned object exists, no mutators on the main graph are allowed. */
     virtual std::unique_ptr<BlockBuilder> GetBlockBuilder() noexcept = 0;
+    /** Construct an evictor, drawing from the main graph, which cannot be oversized. While
+     *  the returned object exists, no mutators on the main graph are allowed, except destroying
+     *  the Refs reported by Evictor::GetCurrentChunk */
+    virtual std::unique_ptr<Evictor> GetEvictor() noexcept = 0;
 
     /** Perform an internal consistency check on this object. */
     virtual void SanityCheck() const = 0;