Merge f27760012b into 66aa6a47bd

7c123c08dd  miner: add package feerate vector to CBlockTemplate (ismaelsadeeq)

Pull request description:

  This PR enables `BlockAssembler` to add all selected packages' fee and virtual size to a vector, and then return the vector as a member of `CBlockTemplate` struct.

  This PR is the first step in the https://github.com/bitcoin/bitcoin/issues/30392 project.

  The packages' vsize and fee are used in #30157 to select a percentile fee rate of the top block in the mempool.

ACKs for top commit:
  rkrux:
    tACK 7c123c08dd
  ryanofsky:
    Code review ACK 7c123c08dd. Changes since last review are rebasing due to a test conflict, giving the new field a better name and description, resolving the test conflict, and renaming a lot of test variables. The actual code change is still one-line change.
  glozow:
    reACK 7c123c08dd

Tree-SHA512: 767b0b3d4273cf1589fd2068d729a66c7414c0f9574b15989fbe293f8c85cd6c641dd783cde55bfabab32cd047d7d8a071d6897b06ed4295c0d071e588de0861

.cirrus.yml

@@ -211,3 +211,13 @@ task:
       type: small
   env:
     FILE_ENV: "./ci/test/00_setup_env_mac_cross.sh"
+
+task:
+  name: 'riscv32 bare metal, static libbitcoin_consensus'
+  << : *GLOBAL_TASK_TEMPLATE
+  persistent_worker:
+    labels:
+      type: small
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_riscv_bare_cross.sh"
+

CMakeLists.txt

@@ -329,11 +329,13 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
   endif()
 endif()
 
+if(NOT CMAKE_SYSTEM_NAME STREQUAL "Generic")
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
 target_link_libraries(core_interface INTERFACE
   Threads::Threads
 )
+endif()
 
 add_library(sanitize_interface INTERFACE)
 target_link_libraries(core_interface INTERFACE sanitize_interface)
@@ -381,8 +383,10 @@ if(BUILD_FUZZ_BINARY)
   )
 endif()
 
+if(NOT CMAKE_SYSTEM_NAME STREQUAL "Generic")
 include(AddBoostIfNeeded)
 add_boost_if_needed()
+endif()
 
 if(BUILD_DAEMON OR BUILD_GUI OR BUILD_CLI OR BUILD_TESTS OR BUILD_BENCH OR BUILD_FUZZ_BINARY)
   find_package(Libevent 2.1.8 MODULE REQUIRED)

ci/test/00_setup_env_riscv_bare_cross.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019-present The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_native_riscv_bare
+
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
+export PACKAGES="autoconf automake autotools-dev curl python3 python3-pip libmpc-dev libmpfr-dev libgmp-dev gawk build-essential bison flex texinfo gperf libtool patchutils bc zlib1g-dev libexpat-dev ninja-build git cmake libglib2.0-dev libslirp-dev"
+export BITCOIN_CONFIG="-DCMAKE_C_COMPILER=/opt/riscv-ilp32/bin/riscv32-unknown-elf-gcc \
+ -DCMAKE_CXX_COMPILER=/opt/riscv-ilp32/bin/riscv32-unknown-elf-g++ \
+ -DBUILD_KERNEL_LIB=OFF \
+ -DBUILD_UTIL_CHAINSTATE=OFF \
+ -DBUILD_TESTS=OFF \
+ -DBUILD_BENCH=OFF \
+ -DBUILD_FUZZ_BINARY=OFF \
+ -DBUILD_DAEMON=OFF \
+ -DBUILD_TX=OFF \
+ -DBUILD_UTIL=OFF \
+ -DBUILD_CLI=OFF \
+ -DENABLE_WALLET=OFF \
+ -DCMAKE_SYSTEM_NAME=Generic \
+ "
+
+export BARE_METAL_RISCV="true"
+export RUN_UNIT_TESTS="false"
+export RUN_FUNCTIONAL_TESTS="false"
+export NO_DEPENDS="true"

ci/test/01_base_install.sh

@@ -84,6 +84,29 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
   rm -rf /msan/llvm-project
 fi
 
+if [[ ${BARE_METAL_RISCV} == "true" ]]; then
+    ${CI_RETRY_EXE} git clone --depth=1 https://github.com/riscv-collab/riscv-gnu-toolchain -b 2024.11.22 /riscv/gcc
+    ( cd /riscv/gcc;
+      ./configure --prefix=/opt/riscv-ilp32 --with-arch=rv32gc --with-abi=ilp32;
+      make -j "$MAKEJOBS";
+      make install; )
+    rm -rf /riscv/gcc
+
+    ${CI_RETRY_EXE} git clone --depth=1 https://sourceware.org/git/newlib-cygwin.git -b topic/3.6 /riscv/newlib
+    ( cd /riscv/newlib;
+      mkdir build && cd build;
+      ../configure \
+          --target=riscv32-unknown-elf --with-arch=rv32gc --with-abi=ilp32 --disable-shared --disable-multilib\
+          --prefix=/opt/newlib \
+          CC_FOR_TARGET=/opt/riscv-ilp32/bin/riscv32-unknown-elf-gcc \
+          CXX_FOR_TARGET=/opt/riscv-ilp32/bin/riscv32-unknown-elf-g++ \
+          AR_FOR_TARGET=/opt/riscv-ilp32/bin/riscv32-unknown-elf-ar \
+          RANLIB_FOR_TARGET=/opt/riscv-ilp32/bin/riscv32-unknown-elf-ranlib
+      make -j "$MAKEJOBS";
+      make install; )
+    rm -rf /riscv/newlib
+fi
+
 if [[ "${RUN_TIDY}" == "true" ]]; then
   ${CI_RETRY_EXE} git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_"${TIDY_LLVM_V}" /include-what-you-use
   cmake -B /iwyu-build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-"${TIDY_LLVM_V}" -S /include-what-you-use

ci/test/03_test_script.sh

@@ -134,6 +134,61 @@ bash -c "${PRINT_CCACHE_STATISTICS}"
 du -sh "${DEPENDS_DIR}"/*/
 du -sh "${PREVIOUS_RELEASES_DIR}"
 
+if [[ ${BARE_METAL_RISCV} == "true" ]]; then
+    cmake --build . --target bitcoin_consensus
+
+    echo -e "#include <script/script_error.h>\n int main() { return ScriptErrorString(ScriptError_t::SCRIPT_ERR_UNKNOWN_ERROR).size() > 0; }" > test.cpp
+
+    /opt/riscv-ilp32/bin/riscv32-unknown-elf-g++ -I "${BASE_ROOT_DIR}"/src -g -std=c++20 -march=rv32gc -mabi=ilp32 -c test.cpp -o test.o
+
+    echo -e ".section .text
+          .global _start
+          .type _start, @function
+
+          _start:
+              .option push
+              .option norelax
+              la gp, __global_pointer$
+              .option pop
+
+              call main
+
+              # Put Exit2 system call number into the a7 register
+              li a7, 93
+              ecall" > start.s
+
+    /opt/riscv-ilp32/bin/riscv32-unknown-elf-gcc -c start.s -o start.o
+
+    echo -e "#include <sys/stat.h>
+          void _exit(int code) { while(1); }
+          int _sbrk(int incr) { return 0; }
+          int _write(int file, char *ptr, int len) { return 0; }
+          int _close(int file) { return -1; }
+          int _fstat(int file, struct stat *st) { st->st_mode = S_IFCHR; return 0; }
+          int _isatty(int file) { return 1; }
+          int _lseek(int file, int ptr, int dir) { return 0; }
+          int _read(int file, char *ptr, int len) { return 0; }
+          int _kill(int pid, int sig) { return -1; }
+          int _getpid(void) { return -1; }" > syscalls.c
+
+    /opt/riscv-ilp32/bin/riscv32-unknown-elf-gcc -g -march=rv32i -mabi=ilp32 -c syscalls.c -o syscalls.o
+
+    /opt/riscv-ilp32/bin/riscv32-unknown-elf-g++ -g -std=c++20 -march=rv32gc -mabi=ilp32 \
+        -nostdlib \
+        /opt/riscv-ilp32/lib/gcc/riscv32-unknown-elf/14.2.0/crtbegin.o \
+        test.o \
+        start.o \
+        syscalls.o \
+        src/libbitcoin_consensus.a \
+        src/crypto/libbitcoin_crypto.a \
+        src/secp256k1/lib/libsecp256k1.a \
+        /opt/riscv-ilp32/riscv32-unknown-elf/lib/libstdc++.a \
+        /opt/newlib/riscv32-unknown-elf/lib/libc.a \
+        /opt/newlib/riscv32-unknown-elf/lib/libm.a \
+        /opt/riscv-ilp32/lib/gcc/riscv32-unknown-elf/14.2.0/libgcc.a \
+        -o test.elf
+fi
+
 if [[ $HOST = *-mingw32 ]]; then
   "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh"
 fi

cmake/introspection.cmake

@@ -26,7 +26,9 @@ if(HAVE_SYS_TYPES_H AND HAVE_IFADDRS_H)
 endif()
 
 include(TestAppendRequiredLibraries)
+if(NOT CMAKE_SYSTEM_NAME STREQUAL "Generic")
 test_append_atomic_library(core_interface)
+endif()
 
 check_cxx_symbol_exists(std::system "cstdlib" HAVE_STD_SYSTEM)
 check_cxx_symbol_exists(::_wsystem "stdlib.h" HAVE__WSYSTEM)

src/CMakeLists.txt

@@ -163,7 +163,7 @@ target_link_libraries(bitcoin_common
     bitcoin_util
     univalue
     secp256k1
-    Boost::headers
+    $<TARGET_NAME_IF_EXISTS:Boost::headers>
     $<TARGET_NAME_IF_EXISTS:USDT::headers>
     $<$<PLATFORM_ID:Windows>:ws2_32>
 )
@@ -298,7 +298,7 @@ target_link_libraries(bitcoin_node
     leveldb
     minisketch
     univalue
-    Boost::headers
+    $<TARGET_NAME_IF_EXISTS:Boost::headers>
     $<TARGET_NAME_IF_EXISTS:libevent::core>
     $<TARGET_NAME_IF_EXISTS:libevent::extra>
     $<TARGET_NAME_IF_EXISTS:libevent::pthreads>

src/node/miner.cpp

@@ -421,6 +421,7 @@ void BlockAssembler::addPackageTxs(int& nPackagesSelected, int& nDescendantsUpda
         }
 
         ++nPackagesSelected;
+        pblocktemplate->m_package_feerates.emplace_back(packageFees, static_cast<int32_t>(packageSize));
 
         // Update transactions that depend on each of these
         nDescendantsUpdated += UpdatePackagesForAdded(mempool, ancestors, mapModifiedTx);

src/node/miner.h

@@ -10,6 +10,7 @@
 #include <policy/policy.h>
 #include <primitives/block.h>
 #include <txmempool.h>
+#include <util/feefrac.h>
 
 #include <memory>
 #include <optional>
@@ -39,6 +40,9 @@ struct CBlockTemplate
     std::vector<CAmount> vTxFees;
     std::vector<int64_t> vTxSigOpsCost;
     std::vector<unsigned char> vchCoinbaseCommitment;
+    /* A vector of package fee rates, ordered by the sequence in which
+     * packages are selected for inclusion in the block template.*/
+    std::vector<FeeFrac> m_package_feerates;
 };
 
 // Container for tracking updates to ancestor feerate as we include (parent)

src/test/miner_tests.cpp

@@ -16,6 +16,7 @@
 #include <txmempool.h>
 #include <uint256.h>
 #include <util/check.h>
+#include <util/feefrac.h>
 #include <util/strencodings.h>
 #include <util/time.h>
 #include <util/translation.h>
@@ -25,6 +26,7 @@
 #include <test/util/setup_common.h>
 
 #include <memory>
+#include <vector>
 
 #include <boost/test/unit_test.hpp>
 
@@ -123,19 +125,22 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const
     tx.vout[0].nValue = 5000000000LL - 1000;
     // This tx has a low fee: 1000 satoshis
     Txid hashParentTx = tx.GetHash(); // save this txid for later use
-    AddToMempool(tx_mempool, entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx));
+    const auto parent_tx{entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)};
+    AddToMempool(tx_mempool, parent_tx);
 
     // This tx has a medium fee: 10000 satoshis
     tx.vin[0].prevout.hash = txFirst[1]->GetHash();
     tx.vout[0].nValue = 5000000000LL - 10000;
     Txid hashMediumFeeTx = tx.GetHash();
-    AddToMempool(tx_mempool, entry.Fee(10000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx));
+    const auto medium_fee_tx{entry.Fee(10000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)};
+    AddToMempool(tx_mempool, medium_fee_tx);
 
     // This tx has a high fee, but depends on the first transaction
     tx.vin[0].prevout.hash = hashParentTx;
     tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 50k satoshi fee
     Txid hashHighFeeTx = tx.GetHash();
-    AddToMempool(tx_mempool, entry.Fee(50000).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx));
+    const auto high_fee_tx{entry.Fee(50000).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)};
+    AddToMempool(tx_mempool, high_fee_tx);
 
     std::unique_ptr<BlockTemplate> block_template = mining->createNewBlock(options);
     BOOST_REQUIRE(block_template);
@@ -145,6 +150,21 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const
     BOOST_CHECK(block.vtx[2]->GetHash() == hashHighFeeTx);
     BOOST_CHECK(block.vtx[3]->GetHash() == hashMediumFeeTx);
 
+    // Test the inclusion of package feerates in the block template and ensure they are sequential.
+    const auto block_package_feerates = BlockAssembler{m_node.chainman->ActiveChainstate(), &tx_mempool, options}.CreateNewBlock()->m_package_feerates;
+    BOOST_CHECK(block_package_feerates.size() == 2);
+
+    // parent_tx and high_fee_tx are added to the block as a package.
+    const auto combined_txs_fee = parent_tx.GetFee() + high_fee_tx.GetFee();
+    const auto combined_txs_size = parent_tx.GetTxSize() + high_fee_tx.GetTxSize();
+    FeeFrac package_feefrac{combined_txs_fee, combined_txs_size};
+    // The package should be added first.
+    BOOST_CHECK(block_package_feerates[0] == package_feefrac);
+
+    // The medium_fee_tx should be added next.
+    FeeFrac medium_tx_feefrac{medium_fee_tx.GetFee(), medium_fee_tx.GetTxSize()};
+    BOOST_CHECK(block_package_feerates[1] == medium_tx_feefrac);
+
     // Test that a package below the block min tx fee doesn't get included
     tx.vin[0].prevout.hash = hashHighFeeTx;
     tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 0 fee

src/test/util/CMakeLists.txt

@@ -23,7 +23,7 @@ add_library(test_util STATIC EXCLUDE_FROM_ALL
 target_link_libraries(test_util
   PRIVATE
     core_interface
-    Boost::headers
+    $<TARGET_NAME_IF_EXISTS:Boost::headers>
   PUBLIC
     univalue
 )