Merge d838f1bac6 into 66aa6a47bd

7c123c08dd  miner: add package feerate vector to CBlockTemplate (ismaelsadeeq)

Pull request description:

  This PR enables `BlockAssembler` to add all selected packages' fee and virtual size to a vector, and then return the vector as a member of `CBlockTemplate` struct.

  This PR is the first step in the https://github.com/bitcoin/bitcoin/issues/30392 project.

  The packages' vsize and fee are used in #30157 to select a percentile fee rate of the top block in the mempool.

ACKs for top commit:
  rkrux:
    tACK 7c123c08dd
  ryanofsky:
    Code review ACK 7c123c08dd. Changes since last review are rebasing due to a test conflict, giving the new field a better name and description, resolving the test conflict, and renaming a lot of test variables. The actual code change is still one-line change.
  glozow:
    reACK 7c123c08dd

Tree-SHA512: 767b0b3d4273cf1589fd2068d729a66c7414c0f9574b15989fbe293f8c85cd6c641dd783cde55bfabab32cd047d7d8a071d6897b06ed4295c0d071e588de0861

ci/test/03_test_script.sh

@@ -172,6 +172,7 @@ if [ "${RUN_TIDY}" = "true" ]; then
     echo "^^^ ⚠️ Failure generated from clang-tidy"
     false
   fi
+
   # Filter out:
   # * qt qrc and moc generated files
   jq 'map(select(.file | test("src/qt/qrc_.*\\.cpp$|/moc_.*\\.cpp$") | not))' "${BASE_BUILD_DIR}/compile_commands.json" > tmp.json
@@ -182,6 +183,13 @@ if [ "${RUN_TIDY}" = "true" ]; then
            -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_ROOT_DIR}/contrib/devtools/iwyu/bitcoin.core.imp" \
            -Xiwyu --max_line_length=160 \
            2>&1 | tee /tmp/iwyu_ci.out
+
+  # TODO: Expand the following list to all targets.
+  TARGETS_WITH_FORCED_IWYU="bitcoin_clientversion bitcoin_crypto"
+  cd "${BASE_BUILD_DIR}"
+  bash -c "cmake -S ${BASE_ROOT_DIR} -DCMAKE_CXX_INCLUDE_WHAT_YOU_USE=\"include-what-you-use;-Xiwyu;--cxx17ns;-Xiwyu;--mapping_file=${BASE_ROOT_DIR}/contrib/devtools/iwyu/bitcoin.core.imp;-Xiwyu;--error\""
+  bash -c "cmake --build . ${MAKEJOBS} --clean-first --target ${TARGETS_WITH_FORCED_IWYU}"
+
   cd "${BASE_ROOT_DIR}/src"
   python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out
   git --no-pager diff

contrib/devtools/iwyu/bitcoin.core.imp

@@ -1,3 +1,11 @@
-# Nothing for now.
 [
+  # Compiler intrinsics
+  { include: [ "<emmintrin.h>", private, "<immintrin.h>", public ] },
+  { include: [ "<smmintrin.h>", private, "<immintrin.h>", public ] },
+  { include: [ "<tmmintrin.h>", private, "<immintrin.h>", public ] },
+
+  # False positive warnings
+  # See: https://github.com/include-what-you-use/include-what-you-use/issues/1616
+  { include: [ "<iterator>", public, "<utility>", public ] },
+  { include: [ "<tuple>", public, "<array>", public ] },
 ]

src/crypto/chacha20.cpp

@@ -12,7 +12,7 @@
 
 #include <algorithm>
 #include <bit>
-#include <string.h>
+#include <cassert>
 
 #define QUARTERROUND(a,b,c,d) \
   a += b; d = std::rotl(d ^ a, 16); \

src/crypto/chacha20.h

@@ -9,7 +9,6 @@
 
 #include <array>
 #include <cstddef>
-#include <cstdlib>
 #include <stdint.h>
 #include <utility>
 

src/crypto/hex_base.cpp

@@ -4,7 +4,10 @@
 
 #include <crypto/hex_base.h>
 
+#include <span.h>
+
 #include <array>
+#include <cassert>
 #include <cstring>
 #include <string>
 

src/crypto/hkdf_sha256_32.cpp

@@ -4,8 +4,9 @@
 
 #include <crypto/hkdf_sha256_32.h>
 
-#include <assert.h>
-#include <string.h>
+#include <crypto/hmac_sha256.h>
+
+#include <cassert>
 
 CHKDF_HMAC_SHA256_L32::CHKDF_HMAC_SHA256_L32(const unsigned char* ikm, size_t ikmlen, const std::string& salt)
 {

src/crypto/hkdf_sha256_32.h

@@ -5,10 +5,8 @@
 #ifndef BITCOIN_CRYPTO_HKDF_SHA256_32_H
 #define BITCOIN_CRYPTO_HKDF_SHA256_32_H
 
-#include <crypto/hmac_sha256.h>
-
 #include <cstdlib>
-#include <stdint.h>
+#include <string>
 
 /** A rfc5869 HKDF implementation with HMAC_SHA256 and fixed key output length of 32 bytes (L=32) */
 class CHKDF_HMAC_SHA256_L32

src/crypto/hmac_sha256.cpp

@@ -4,6 +4,7 @@
 
 #include <crypto/hmac_sha256.h>
 
+#include <crypto/sha256.h>
 #include <string.h>
 
 CHMAC_SHA256::CHMAC_SHA256(const unsigned char* key, size_t keylen)

src/crypto/hmac_sha256.h

@@ -8,7 +8,6 @@
 #include <crypto/sha256.h>
 
 #include <cstdlib>
-#include <stdint.h>
 
 /** A hasher class for HMAC-SHA-256. */
 class CHMAC_SHA256

src/crypto/hmac_sha512.cpp

@@ -4,6 +4,7 @@
 
 #include <crypto/hmac_sha512.h>
 
+#include <crypto/sha512.h>
 #include <string.h>
 
 CHMAC_SHA512::CHMAC_SHA512(const unsigned char* key, size_t keylen)

src/crypto/hmac_sha512.h

@@ -8,7 +8,6 @@
 #include <crypto/sha512.h>
 
 #include <cstdlib>
-#include <stdint.h>
 
 /** A hasher class for HMAC-SHA-512. */
 class CHMAC_SHA512

src/crypto/muhash.cpp

@@ -7,9 +7,10 @@
 #include <crypto/chacha20.h>
 #include <crypto/common.h>
 #include <hash.h>
+#include <span.h>
+#include <uint256.h>
 
 #include <cassert>
-#include <cstdio>
 #include <limits>
 
 namespace {

src/crypto/muhash.h

@@ -6,9 +6,13 @@
 #define BITCOIN_CRYPTO_MUHASH_H
 
 #include <serialize.h>
-#include <uint256.h>
 
-#include <stdint.h>
+#include <cstddef>
+#include <cstdint>
+
+class uint256;
+template <typename C>
+class Span;
 
 class Num3072
 {

src/crypto/poly1305.cpp

@@ -5,8 +5,6 @@
 #include <crypto/common.h>
 #include <crypto/poly1305.h>
 
-#include <string.h>
-
 namespace poly1305_donna {
 
 // Based on the public domain implementation by Andrew Moon

src/crypto/poly1305.h

@@ -8,7 +8,7 @@
 #include <span.h>
 
 #include <cassert>
-#include <cstdlib>
+#include <cstddef>
 #include <stdint.h>
 
 #define POLY1305_BLOCK_SIZE 16

src/crypto/sha256_x86_shani.cpp

@@ -8,7 +8,8 @@
 
 #if defined(ENABLE_SSE41) && defined(ENABLE_X86_SHANI)
 
-#include <stdint.h>
+#include <cstddef>
+#include <cstdint>
 #include <immintrin.h>
 
 #include <attributes.h>

src/crypto/sha3.cpp

@@ -10,10 +10,9 @@
 #include <span.h>
 
 #include <algorithm>
-#include <array> // For std::begin and std::end.
 #include <bit>
-
-#include <stdint.h>
+#include <cassert>
+#include <iterator>
 
 void KeccakF(uint64_t (&st)[25])
 {

src/crypto/sha3.h

@@ -5,10 +5,11 @@
 #ifndef BITCOIN_CRYPTO_SHA3_H
 #define BITCOIN_CRYPTO_SHA3_H
 
-#include <span.h>
-
+#include <cstdint>
 #include <cstdlib>
-#include <stdint.h>
+
+template <typename C>
+class Span;
 
 //! The Keccak-f[1600] transform.
 void KeccakF(uint64_t (&st)[25]);

src/crypto/siphash.cpp

@@ -4,7 +4,11 @@
 
 #include <crypto/siphash.h>
 
+#include <span.h>
+#include <uint256.h>
+
 #include <bit>
+#include <cassert>
 
 #define SIPROUND do { \
     v0 += v1; v1 = std::rotl(v1, 13); v1 ^= v0; \

src/crypto/siphash.h

@@ -7,8 +7,9 @@
 
 #include <stdint.h>
 
-#include <span.h>
-#include <uint256.h>
+class uint256;
+template <typename C>
+class Span;
 
 /** SipHash-2-4 */
 class CSipHasher

src/node/miner.cpp

@@ -421,6 +421,7 @@ void BlockAssembler::addPackageTxs(int& nPackagesSelected, int& nDescendantsUpda
         }
 
         ++nPackagesSelected;
+        pblocktemplate->m_package_feerates.emplace_back(packageFees, static_cast<int32_t>(packageSize));
 
         // Update transactions that depend on each of these
         nDescendantsUpdated += UpdatePackagesForAdded(mempool, ancestors, mapModifiedTx);

src/node/miner.h

@@ -10,6 +10,7 @@
 #include <policy/policy.h>
 #include <primitives/block.h>
 #include <txmempool.h>
+#include <util/feefrac.h>
 
 #include <memory>
 #include <optional>
@@ -39,6 +40,9 @@ struct CBlockTemplate
     std::vector<CAmount> vTxFees;
     std::vector<int64_t> vTxSigOpsCost;
     std::vector<unsigned char> vchCoinbaseCommitment;
+    /* A vector of package fee rates, ordered by the sequence in which
+     * packages are selected for inclusion in the block template.*/
+    std::vector<FeeFrac> m_package_feerates;
 };
 
 // Container for tracking updates to ancestor feerate as we include (parent)

src/test/miner_tests.cpp

@@ -16,6 +16,7 @@
 #include <txmempool.h>
 #include <uint256.h>
 #include <util/check.h>
+#include <util/feefrac.h>
 #include <util/strencodings.h>
 #include <util/time.h>
 #include <util/translation.h>
@@ -25,6 +26,7 @@
 #include <test/util/setup_common.h>
 
 #include <memory>
+#include <vector>
 
 #include <boost/test/unit_test.hpp>
 
@@ -123,19 +125,22 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const
     tx.vout[0].nValue = 5000000000LL - 1000;
     // This tx has a low fee: 1000 satoshis
     Txid hashParentTx = tx.GetHash(); // save this txid for later use
-    AddToMempool(tx_mempool, entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx));
+    const auto parent_tx{entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)};
+    AddToMempool(tx_mempool, parent_tx);
 
     // This tx has a medium fee: 10000 satoshis
     tx.vin[0].prevout.hash = txFirst[1]->GetHash();
     tx.vout[0].nValue = 5000000000LL - 10000;
     Txid hashMediumFeeTx = tx.GetHash();
-    AddToMempool(tx_mempool, entry.Fee(10000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx));
+    const auto medium_fee_tx{entry.Fee(10000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)};
+    AddToMempool(tx_mempool, medium_fee_tx);
 
     // This tx has a high fee, but depends on the first transaction
     tx.vin[0].prevout.hash = hashParentTx;
     tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 50k satoshi fee
     Txid hashHighFeeTx = tx.GetHash();
-    AddToMempool(tx_mempool, entry.Fee(50000).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx));
+    const auto high_fee_tx{entry.Fee(50000).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)};
+    AddToMempool(tx_mempool, high_fee_tx);
 
     std::unique_ptr<BlockTemplate> block_template = mining->createNewBlock(options);
     BOOST_REQUIRE(block_template);
@@ -145,6 +150,21 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const
     BOOST_CHECK(block.vtx[2]->GetHash() == hashHighFeeTx);
     BOOST_CHECK(block.vtx[3]->GetHash() == hashMediumFeeTx);
 
+    // Test the inclusion of package feerates in the block template and ensure they are sequential.
+    const auto block_package_feerates = BlockAssembler{m_node.chainman->ActiveChainstate(), &tx_mempool, options}.CreateNewBlock()->m_package_feerates;
+    BOOST_CHECK(block_package_feerates.size() == 2);
+
+    // parent_tx and high_fee_tx are added to the block as a package.
+    const auto combined_txs_fee = parent_tx.GetFee() + high_fee_tx.GetFee();
+    const auto combined_txs_size = parent_tx.GetTxSize() + high_fee_tx.GetTxSize();
+    FeeFrac package_feefrac{combined_txs_fee, combined_txs_size};
+    // The package should be added first.
+    BOOST_CHECK(block_package_feerates[0] == package_feefrac);
+
+    // The medium_fee_tx should be added next.
+    FeeFrac medium_tx_feefrac{medium_fee_tx.GetFee(), medium_fee_tx.GetTxSize()};
+    BOOST_CHECK(block_package_feerates[1] == medium_tx_feefrac);
+
     // Test that a package below the block min tx fee doesn't get included
     tx.vin[0].prevout.hash = hashHighFeeTx;
     tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 0 fee