Merge bitcoin/bitcoin#27624: [23.2] Backports for rc1

a26ff204f0 doc: add initial release notes for v23.2 (fanquake)
60edfd57f1 doc: update manual pages for v23.2rc1 (fanquake)
b93814b234 doc: update version in bips.md to v23.2 (fanquake)
67bbe6d371 build: bump version to v23.2rc1 (fanquake)
06731d19bc net_processing: Boost inv trickle rate (Anthony Towns)
d0a2c87214 txmempool: have CompareDepthAndScore sort missing txs first (Anthony Towns)
ce8f812b0a p2p: Avoid prematurely clearing download state for other peers (Suhas Daftuar)

Pull request description:

  Final backports for `rc1`. Currently:
  * https://github.com/bitcoin/bitcoin/pull/27608 (not a clean cherry-pick)
  * https://github.com/bitcoin/bitcoin/pull/27610 (second commit is not clean)

ACKs for top commit:
  achow101:
    ACK a26ff204f0
  dergoegge:
    ACK a26ff204f0
  ajtowns:
    utACK a26ff204f0

Tree-SHA512: 59e43ec4d5004b3543d5c0366c9dc8c5f8a6a777b147628ebc0c03aeb0846312a7780376ebf40f389e3403e4501ba2b70bb97925479670bee13c89e5b6925137

configure.ac

@@ -1,8 +1,8 @@
 AC_PREREQ([2.69])
 define(_CLIENT_VERSION_MAJOR, 23)
-define(_CLIENT_VERSION_MINOR, 1)
+define(_CLIENT_VERSION_MINOR, 2)
 define(_CLIENT_VERSION_BUILD, 0)
-define(_CLIENT_VERSION_RC, 0)
+define(_CLIENT_VERSION_RC, 1)
 define(_CLIENT_VERSION_IS_RELEASE, true)
 define(_COPYRIGHT_YEAR, 2022)
 define(_COPYRIGHT_HOLDERS,[The %s developers])

doc/bips.md

@@ -1,4 +1,4 @@
-BIPs that are implemented by Bitcoin Core (up-to-date up to **v23.1**):
+BIPs that are implemented by Bitcoin Core (up-to-date up to **v23.2**):
 
 * [`BIP 9`](https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki): The changes allowing multiple soft-forks to be deployed in parallel have been implemented since **v0.12.1**  ([PR #7575](https://github.com/bitcoin/bitcoin/pull/7575))
 * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)).

doc/man/bitcoin-cli.1

@@ -1,7 +1,7 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.2.
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.3.
-.TH BITCOIN-CLI "1" "December 2022" "bitcoin-cli v23.1.0" "User Commands"
+.TH BITCOIN-CLI "1" "May 2023" "bitcoin-cli v23.2.0rc1" "User Commands"
 .SH NAME
-bitcoin-cli \- manual page for bitcoin-cli v23.1.0
+bitcoin-cli \- manual page for bitcoin-cli v23.2.0rc1
 .SH SYNOPSIS
 .B bitcoin-cli
 [\fI\,options\/\fR] \fI\,<command> \/\fR[\fI\,params\/\fR]  \fI\,Send command to Bitcoin Core\/\fR
@@ -15,7 +15,7 @@ bitcoin-cli \- manual page for bitcoin-cli v23.1.0
 .B bitcoin-cli
 [\fI\,options\/\fR] \fI\,help <command>      Get help for a command\/\fR
 .SH DESCRIPTION
-Bitcoin Core RPC client version v23.1.0
+Bitcoin Core RPC client version v23.2.0rc1
 .SH OPTIONS
 .HP
 \-?

doc/man/bitcoin-qt.1

@@ -1,12 +1,12 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.2.
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.3.
-.TH BITCOIN-QT "1" "December 2022" "bitcoin-qt v23.1.0" "User Commands"
+.TH BITCOIN-QT "1" "May 2023" "bitcoin-qt v23.2.0rc1" "User Commands"
 .SH NAME
-bitcoin-qt \- manual page for bitcoin-qt v23.1.0
+bitcoin-qt \- manual page for bitcoin-qt v23.2.0rc1
 .SH SYNOPSIS
 .B bitcoin-qt
 [\fI\,command-line options\/\fR]
 .SH DESCRIPTION
-Bitcoin Core version v23.1.0
+Bitcoin Core version v23.2.0rc1
 .SH OPTIONS
 .HP
 \-?
@@ -115,7 +115,7 @@ Do not keep transactions in the mempool longer than <n> hours (default:
 .HP
 \fB\-par=\fR<n>
 .IP
-Set the number of script verification threads (\fB\-6\fR to 15, 0 = auto, <0 =
+Set the number of script verification threads (\fB\-10\fR to 15, 0 = auto, <0 =
 leave that many cores free, default: 0)
 .HP
 \fB\-persistmempool\fR

doc/man/bitcoin-tx.1

@@ -1,7 +1,7 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.2.
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.3.
-.TH BITCOIN-TX "1" "December 2022" "bitcoin-tx v23.1.0" "User Commands"
+.TH BITCOIN-TX "1" "May 2023" "bitcoin-tx v23.2.0rc1" "User Commands"
 .SH NAME
-bitcoin-tx \- manual page for bitcoin-tx v23.1.0
+bitcoin-tx \- manual page for bitcoin-tx v23.2.0rc1
 .SH SYNOPSIS
 .B bitcoin-tx
 [\fI\,options\/\fR] \fI\,<hex-tx> \/\fR[\fI\,commands\/\fR]  \fI\,Update hex-encoded bitcoin transaction\/\fR
@@ -9,7 +9,7 @@ bitcoin-tx \- manual page for bitcoin-tx v23.1.0
 .B bitcoin-tx
 [\fI\,options\/\fR] \fI\,-create \/\fR[\fI\,commands\/\fR]   \fI\,Create hex-encoded bitcoin transaction\/\fR
 .SH DESCRIPTION
-Bitcoin Core bitcoin\-tx utility version v23.1.0
+Bitcoin Core bitcoin\-tx utility version v23.2.0rc1
 .SH OPTIONS
 .HP
 \-?

doc/man/bitcoin-util.1

@@ -1,12 +1,12 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.2.
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.3.
-.TH BITCOIN-UTIL "1" "December 2022" "bitcoin-util v23.1.0" "User Commands"
+.TH BITCOIN-UTIL "1" "May 2023" "bitcoin-util v23.2.0rc1" "User Commands"
 .SH NAME
-bitcoin-util \- manual page for bitcoin-util v23.1.0
+bitcoin-util \- manual page for bitcoin-util v23.2.0rc1
 .SH SYNOPSIS
 .B bitcoin-util
 [\fI\,options\/\fR] [\fI\,commands\/\fR]  \fI\,Do stuff\/\fR
 .SH DESCRIPTION
-Bitcoin Core bitcoin\-util utility version v23.1.0
+Bitcoin Core bitcoin\-util utility version v23.2.0rc1
 .SH OPTIONS
 .HP
 \-?

doc/man/bitcoin-wallet.1

@@ -1,9 +1,9 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.2.
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.3.
-.TH BITCOIN-WALLET "1" "December 2022" "bitcoin-wallet v23.1.0" "User Commands"
+.TH BITCOIN-WALLET "1" "May 2023" "bitcoin-wallet v23.2.0rc1" "User Commands"
 .SH NAME
-bitcoin-wallet \- manual page for bitcoin-wallet v23.1.0
+bitcoin-wallet \- manual page for bitcoin-wallet v23.2.0rc1
 .SH DESCRIPTION
-Bitcoin Core bitcoin\-wallet version v23.1.0
+Bitcoin Core bitcoin\-wallet version v23.2.0rc1
 .PP
 bitcoin\-wallet is an offline tool for creating and interacting with Bitcoin Core wallet files.
 By default bitcoin\-wallet will act on wallets in the default mainnet wallet directory in the datadir.

doc/man/bitcoind.1

@@ -1,12 +1,12 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.2.
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.3.
-.TH BITCOIND "1" "December 2022" "bitcoind v23.1.0" "User Commands"
+.TH BITCOIND "1" "May 2023" "bitcoind v23.2.0rc1" "User Commands"
 .SH NAME
-bitcoind \- manual page for bitcoind v23.1.0
+bitcoind \- manual page for bitcoind v23.2.0rc1
 .SH SYNOPSIS
 .B bitcoind
 [\fI\,options\/\fR]                     \fI\,Start Bitcoin Core\/\fR
 .SH DESCRIPTION
-Bitcoin Core version v23.1.0
+Bitcoin Core version v23.2.0rc1
 .SH OPTIONS
 .HP
 \-?
@@ -115,7 +115,7 @@ Do not keep transactions in the mempool longer than <n> hours (default:
 .HP
 \fB\-par=\fR<n>
 .IP
-Set the number of script verification threads (\fB\-6\fR to 15, 0 = auto, <0 =
+Set the number of script verification threads (\fB\-10\fR to 15, 0 = auto, <0 =
 leave that many cores free, default: 0)
 .HP
 \fB\-persistmempool\fR

doc/release-notes.md

@@ -1,9 +1,9 @@
-23.1 Release Notes
+23.2rc1 Release Notes
 ==================
 
-Bitcoin Core version 23.1 is now available from:
+Bitcoin Core version 23.2rc1 is now available from:
 
-  <https://bitcoincore.org/bin/bitcoin-core-23.1/>
+  <https://bitcoincore.org/bin/bitcoin-core-23.2/test.rc1/>
 
 This release includes new features, various bug fixes and performance
 improvements, as well as updated translations.
@@ -39,52 +39,34 @@ unsupported systems.
 
 ### P2P
 
-- #25314 p2p: always set nTime for self-advertisements
+- #26909 net: prevent peers.dat corruptions by only serializing once
-
+- #27608 p2p: Avoid prematurely clearing download state for other peers
-### RPC and other APIs
+- #27610 Improve performance of p2p inv to send queues
-
-- #25220 rpc: fix incorrect warning for address type p2sh-segwit in createmultisig
-- #25237 rpc: Capture UniValue by ref for rpcdoccheck
-- #25983 Prevent data race for pathHandlers
-- #26275 Fix crash on deriveaddresses when index is 2147483647 (2^31-1)
 
 ### Build system
 
-- #25201 windeploy: Renewed windows code signing certificate
+- #25436 build: suppress array-bounds errors in libxkbcommon
-- #25788 guix: patch NSIS to remove .reloc sections from installer stubs
+- #25763 bdb: disable Werror for format-security
-- #25861 guix: use --build={arch}-guix-linux-gnu in cross toolchain
+- #26944 depends: fix systemtap download URL
-- #25985 Revert "build: Use Homebrew's sqlite package if it is available"
+- #27462 depends: fix compiling bdb with clang-16 on aarch64
-
-### GUI
-
-- #24668 build, qt: bump Qt5 version to 5.15.3
-- gui#631 Disallow encryption of watchonly wallets
-- gui#680 Fixes MacOS 13 segfault by preventing certain notifications
-
-### Tests
-
-- #24454 tests: Fix calculation of external input weights
 
 ### Miscellaneous
 
-- #26321 Adjust .tx/config for new Transifex CLI
+- #25444 ci: macOS task imrovements
+- #26388 ci: Use macos-ventura-xcode:14.1 image for "macOS native" task
+- #26924 refactor: Add missing includes to fix gcc-13 compile error
 
 Credits
 =======
 
 Thanks to everyone who directly contributed to this release:
 
-- Andrew Chow
+- Anthony Towns
-- brunoerg
 - Hennadii Stepanov
-- John Moffett
 - MacroFake
 - Martin Zumsande
 - Michael Ford
-- muxator
+- Suhas Daftuar
-- Pavol Rusnak
-- Sebastian Falbesoner
-- W. J. van der Laan
 
 As well as to everyone that helped with translations on
 [Transifex](https://www.transifex.com/bitcoin/bitcoin/).

doc/release-notes/release-notes-23.1.md

@@ -0,0 +1,90 @@
+23.1 Release Notes
+==================
+
+Bitcoin Core version 23.1 is now available from:
+
+  <https://bitcoincore.org/bin/bitcoin-core-23.1/>
+
+This release includes new features, various bug fixes and performance
+improvements, as well as updated translations.
+
+Please report bugs using the issue tracker at GitHub:
+
+  <https://github.com/bitcoin/bitcoin/issues>
+
+To receive security and update notifications, please subscribe to:
+
+  <https://bitcoincore.org/en/list/announcements/join/>
+
+How to Upgrade
+==============
+
+If you are running an older version, shut it down. Wait until it has completely
+shut down (which might take a few minutes in some cases), then run the
+installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS)
+or `bitcoind`/`bitcoin-qt` (on Linux).
+
+Upgrading directly from a version of Bitcoin Core that has reached its EOL is
+possible, but it might take some time if the data directory needs to be migrated. Old
+wallet versions of Bitcoin Core are generally supported.
+
+Compatibility
+==============
+
+Bitcoin Core is supported and extensively tested on operating systems
+using the Linux kernel, macOS 10.15+, and Windows 7 and newer.  Bitcoin
+Core should also work on most other Unix-like systems but is not as
+frequently tested on them.  It is not recommended to use Bitcoin Core on
+unsupported systems.
+
+### P2P
+
+- #25314 p2p: always set nTime for self-advertisements
+
+### RPC and other APIs
+
+- #25220 rpc: fix incorrect warning for address type p2sh-segwit in createmultisig
+- #25237 rpc: Capture UniValue by ref for rpcdoccheck
+- #25983 Prevent data race for pathHandlers
+- #26275 Fix crash on deriveaddresses when index is 2147483647 (2^31-1)
+
+### Build system
+
+- #25201 windeploy: Renewed windows code signing certificate
+- #25788 guix: patch NSIS to remove .reloc sections from installer stubs
+- #25861 guix: use --build={arch}-guix-linux-gnu in cross toolchain
+- #25985 Revert "build: Use Homebrew's sqlite package if it is available"
+
+### GUI
+
+- #24668 build, qt: bump Qt5 version to 5.15.3
+- gui#631 Disallow encryption of watchonly wallets
+- gui#680 Fixes MacOS 13 segfault by preventing certain notifications
+
+### Tests
+
+- #24454 tests: Fix calculation of external input weights
+
+### Miscellaneous
+
+- #26321 Adjust .tx/config for new Transifex CLI
+
+Credits
+=======
+
+Thanks to everyone who directly contributed to this release:
+
+- Andrew Chow
+- brunoerg
+- Hennadii Stepanov
+- John Moffett
+- MacroFake
+- Martin Zumsande
+- Michael Ford
+- muxator
+- Pavol Rusnak
+- Sebastian Falbesoner
+- W. J. van der Laan
+
+As well as to everyone that helped with translations on
+[Transifex](https://www.transifex.com/bitcoin/bitcoin/).

src/net_processing.cpp

@@ -541,8 +541,11 @@ private:
     /** Remove this block from our tracked requested blocks. Called if:
      *  - the block has been received from a peer
      *  - the request for the block has timed out
+     * If "from_peer" is specified, then only remove the block if it is in
+     * flight from that peer (to avoid one peer's network traffic from
+     * affecting another's state).
      */
-    void RemoveBlockRequest(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
+    void RemoveBlockRequest(const uint256& hash, std::optional<NodeId> from_peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main);
 
     /* Mark a block as in flight
      * Returns false, still setting pit, if the block was already in flight from the same peer
@@ -853,7 +856,7 @@ bool PeerManagerImpl::IsBlockRequested(const uint256& hash)
     return mapBlocksInFlight.find(hash) != mapBlocksInFlight.end();
 }
 
-void PeerManagerImpl::RemoveBlockRequest(const uint256& hash)
+void PeerManagerImpl::RemoveBlockRequest(const uint256& hash, std::optional<NodeId> from_peer)
 {
     auto it = mapBlocksInFlight.find(hash);
     if (it == mapBlocksInFlight.end()) {
@@ -862,6 +865,12 @@ void PeerManagerImpl::RemoveBlockRequest(const uint256& hash)
     }
 
     auto [node_id, list_it] = it->second;
+
+    if (from_peer && node_id != *from_peer) {
+        // Block was requested by another peer
+        return;
+    }
+
     CNodeState *state = State(node_id);
     assert(state != nullptr);
 
@@ -897,7 +906,7 @@ bool PeerManagerImpl::BlockRequested(NodeId nodeid, const CBlockIndex& block, st
     }
 
     // Make sure it's not listed somewhere already.
-    RemoveBlockRequest(hash);
+    RemoveBlockRequest(hash, std::nullopt);
 
     std::list<QueuedBlock>::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(),
             {&block, std::unique_ptr<PartiallyDownloadedBlock>(pit ? new PartiallyDownloadedBlock(&m_mempool) : nullptr)});
@@ -2545,6 +2554,11 @@ void PeerManagerImpl::ProcessBlock(CNode& node, const std::shared_ptr<const CBlo
     m_chainman.ProcessNewBlock(m_chainparams, block, force_processing, &new_block);
     if (new_block) {
         node.m_last_block_time = GetTime<std::chrono::seconds>();
+        // In case this block came from a different peer than we requested
+        // from, we can erase the block request now anyway (as we just stored
+        // this block to disk).
+        LOCK(cs_main);
+        RemoveBlockRequest(block->GetHash(), std::nullopt);
     } else {
         LOCK(cs_main);
         mapBlockSource.erase(block->GetHash());
@@ -3604,7 +3618,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type,
                 PartiallyDownloadedBlock& partialBlock = *(*queuedBlockIt)->partialBlock;
                 ReadStatus status = partialBlock.InitData(cmpctblock, vExtraTxnForCompact);
                 if (status == READ_STATUS_INVALID) {
-                    RemoveBlockRequest(pindex->GetBlockHash()); // Reset in-flight state in case Misbehaving does not result in a disconnect
+                    RemoveBlockRequest(pindex->GetBlockHash(), pfrom.GetId()); // Reset in-flight state in case Misbehaving does not result in a disconnect
                     Misbehaving(pfrom.GetId(), 100, "invalid compact block");
                     return;
                 } else if (status == READ_STATUS_FAILED) {
@@ -3699,7 +3713,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type,
                 // process from some other peer.  We do this after calling
                 // ProcessNewBlock so that a malleated cmpctblock announcement
                 // can't be used to interfere with block relay.
-                RemoveBlockRequest(pblock->GetHash());
+                RemoveBlockRequest(pblock->GetHash(), std::nullopt);
             }
         }
         return;
@@ -3731,7 +3745,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type,
             PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock;
             ReadStatus status = partialBlock.FillBlock(*pblock, resp.txn);
             if (status == READ_STATUS_INVALID) {
-                RemoveBlockRequest(resp.blockhash); // Reset in-flight state in case Misbehaving does not result in a disconnect
+                RemoveBlockRequest(resp.blockhash, pfrom.GetId()); // Reset in-flight state in case Misbehaving does not result in a disconnect
                 Misbehaving(pfrom.GetId(), 100, "invalid compact block/non-matching block transactions");
                 return;
             } else if (status == READ_STATUS_FAILED) {
@@ -3757,7 +3771,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type,
                 // though the block was successfully read, and rely on the
                 // handling in ProcessNewBlock to ensure the block index is
                 // updated, etc.
-                RemoveBlockRequest(resp.blockhash); // it is now an empty pointer
+                RemoveBlockRequest(resp.blockhash, pfrom.GetId()); // it is now an empty pointer
                 fBlockRead = true;
                 // mapBlockSource is used for potentially punishing peers and
                 // updating which peers send us compact blocks, so the race
@@ -3825,7 +3839,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type,
             // Always process the block if we requested it, since we may
             // need it even when it's not a candidate for a new best tip.
             forceProcessing = IsBlockRequested(hash);
-            RemoveBlockRequest(hash);
+            RemoveBlockRequest(hash, pfrom.GetId());
             // mapBlockSource is only used for punishing peers and setting
             // which peers send us compact blocks, so the race between here and
             // cs_main in ProcessNewBlock is fine.
@@ -4876,7 +4890,9 @@ bool PeerManagerImpl::SendMessages(CNode* pto)
                     // especially since we have many peers and some will draw much shorter delays.
                     unsigned int nRelayedTransactions = 0;
                     LOCK(pto->m_tx_relay->cs_filter);
-                    while (!vInvTx.empty() && nRelayedTransactions < INVENTORY_BROADCAST_MAX) {
+                    size_t broadcast_max{INVENTORY_BROADCAST_MAX + (pto->m_tx_relay->setInventoryTxToSend.size()/1000)*5};
+                    broadcast_max = std::min<size_t>(1000, broadcast_max);
+                    while (!vInvTx.empty() && nRelayedTransactions < broadcast_max) {
                         // Fetch the top element from the heap
                         std::pop_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder);
                         std::set<uint256>::iterator it = vInvTx.back();

src/txmempool.cpp

@@ -831,11 +831,16 @@ void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendhei
 
 bool CTxMemPool::CompareDepthAndScore(const uint256& hasha, const uint256& hashb, bool wtxid)
 {
+    /* Return `true` if hasha should be considered sooner than hashb. Namely when:
+     *   a is not in the mempool, but b is
+     *   both are in the mempool and a has fewer ancestors than b
+     *   both are in the mempool and a has a higher score than b
+     */
     LOCK(cs);
-    indexed_transaction_set::const_iterator i = wtxid ? get_iter_from_wtxid(hasha) : mapTx.find(hasha);
-    if (i == mapTx.end()) return false;
     indexed_transaction_set::const_iterator j = wtxid ? get_iter_from_wtxid(hashb) : mapTx.find(hashb);
-    if (j == mapTx.end()) return true;
+    if (j == mapTx.end()) return false;
+    indexed_transaction_set::const_iterator i = wtxid ? get_iter_from_wtxid(hasha) : mapTx.find(hasha);
+    if (i == mapTx.end()) return true;
     uint64_t counta = i->GetCountWithAncestors();
     uint64_t countb = j->GetCountWithAncestors();
     if (counta == countb) {