Merge #21560: net: Add Tor v3 hardcoded seeds

b2ee8b207d net: Deserialize hardcoded seeds from BIP155 blob (W. J. van der Laan)
9b29d5df7f contrib: Add explicit port numbers for testnet seeds (W. J. van der Laan)
2a257de113 contrib: Add a few TorV3 seed nodes (W. J. van der Laan)
06030f7a42 contrib: generate-seeds.py generates output in BIP155 format (W. J. van der Laan)

Pull request description:

  Closes #20239 and mitigates my node's problem in #21351.

  - Add a few hardcoded seeds for TorV3
    - As the [bitcoin-seeder](https://github.com/sipa/bitcoin-seeder) doesn't collect TorV3 addresses yet, I have extracted these from my own node using [a script](https://gist.github.com/laanwj/b3d7b01ef61ce07c2eff0a72a6b90183) and added them manually. This is intended to be a temporary stop gap until 22.0's seeds update.

  - Change hardcoded seeds to variable length BIP155 binary format.
    - It is stored as a single serialized blob in a byte array, instead of pseudo-IPv6 address slots. This is more flexible and, assuming most of the list is IPv4, more compact.
    - Only the (networkID, addr, port) subset (CService). Services and time are construed on the fly as before.

  - Change input format for `nodes_*.txt`.
    - Drop legacy `0xAABBCCDD` format for IPv4. It is never generated by `makeseeds.py`.
    - Stop interpreting lack of port as default port, interpret it as 'no port', to accomodate I2P and other port-less protocols (not handled in this PR). An explicit port is always generated by `makeseeds.py` so in practice this makes no difference right now.

  A follow-up to this PR could do the same for I2P.

ACKs for top commit:
  jonatack:
    ACK b2ee8b207d

Tree-SHA512: 11a6b54f9fb0192560f2bd7b218f798f86c1abe01d1bf37f734cb88b91848124beb2de801ca4e6f856e9946aea5dc3ee16b0dbb9863799e42eec1b239d40d59d

contrib/seeds/generate-seeds.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2014-2017 Wladimir J. van der Laan
+# Copyright (c) 2014-2021 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 '''
@@ -13,19 +13,13 @@ argument:
 
 These files must consist of lines in the format
 
-    <ip>
     <ip>:<port>
-    [<ipv6>]
     [<ipv6>]:<port>
-    <onion>.onion
-    0xDDBBCCAA (IPv4 little-endian old pnSeeds format)
+    <onion>.onion:<port>
 
 The output will be two data structures with the peers in binary format:
 
-   static SeedSpec6 pnSeed6_main[]={
-   ...
-   }
-   static SeedSpec6 pnSeed6_test[]={
+   static const uint8_t chainparams_seed_{main,test}[]={
    ...
    }
 
@@ -33,24 +27,33 @@ These should be pasted into `src/chainparamsseeds.h`.
 '''
 
 from base64 import b32decode
-from binascii import a2b_hex
+from enum import Enum
+import struct
 import sys
 import os
 import re
 
-# ipv4 in ipv6 prefix
-pchIPv4 = bytearray([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff])
-# tor-specific ipv6 prefix
-pchOnionCat = bytearray([0xFD,0x87,0xD8,0x7E,0xEB,0x43])
+class BIP155Network(Enum):
+    IPV4 = 1
+    IPV6 = 2
+    TORV2 = 3
+    TORV3 = 4
+    I2P = 5
+    CJDNS = 6
 
-def name_to_ipv6(addr):
-    if len(addr)>6 and addr.endswith('.onion'):
+def name_to_bip155(addr):
+    '''Convert address string to BIP155 (networkID, addr) tuple.'''
+    if addr.endswith('.onion'):
         vchAddr = b32decode(addr[0:-6], True)
-        if len(vchAddr) != 16-len(pchOnionCat):
+        if len(vchAddr) == 10:
+            return (BIP155Network.TORV2, vchAddr)
+        elif len(vchAddr) == 35:
+            assert(vchAddr[34] == 3)
+            return (BIP155Network.TORV3, vchAddr[:32])
+        else:
             raise ValueError('Invalid onion %s' % vchAddr)
-        return pchOnionCat + vchAddr
     elif '.' in addr: # IPv4
-        return pchIPv4 + bytearray((int(x) for x in addr.split('.')))
+        return (BIP155Network.IPV4, bytes((int(x) for x in addr.split('.'))))
     elif ':' in addr: # IPv6
         sub = [[], []] # prefix, suffix
         x = 0
@@ -67,13 +70,12 @@ def name_to_ipv6(addr):
                 sub[x].append(val & 0xff)
         nullbytes = 16 - len(sub[0]) - len(sub[1])
         assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0))
-        return bytearray(sub[0] + ([0] * nullbytes) + sub[1])
-    elif addr.startswith('0x'): # IPv4-in-little-endian
-        return pchIPv4 + bytearray(reversed(a2b_hex(addr[2:])))
+        return (BIP155Network.IPV6, bytes(sub[0] + ([0] * nullbytes) + sub[1]))
     else:
         raise ValueError('Could not parse address %s' % addr)
 
-def parse_spec(s, defaultport):
+def parse_spec(s):
+    '''Convert endpoint string to BIP155 (networkID, addr, port) tuple.'''
     match = re.match(r'\[([0-9a-fA-F:]+)\](?::([0-9]+))?$', s)
     if match: # ipv6
         host = match.group(1)
@@ -85,17 +87,39 @@ def parse_spec(s, defaultport):
         (host,_,port) = s.partition(':')
 
     if not port:
-        port = defaultport
+        port = 0
     else:
         port = int(port)
 
-    host = name_to_ipv6(host)
+    host = name_to_bip155(host)
 
-    return (host,port)
+    return host + (port, )
 
-def process_nodes(g, f, structname, defaultport):
-    g.write('static SeedSpec6 %s[] = {\n' % structname)
-    first = True
+def ser_compact_size(l):
+    r = b""
+    if l < 253:
+        r = struct.pack("B", l)
+    elif l < 0x10000:
+        r = struct.pack("<BH", 253, l)
+    elif l < 0x100000000:
+        r = struct.pack("<BI", 254, l)
+    else:
+        r = struct.pack("<BQ", 255, l)
+    return r
+
+def bip155_serialize(spec):
+    '''
+    Serialize (networkID, addr, port) tuple to BIP155 binary format.
+    '''
+    r = b""
+    r += struct.pack('B', spec[0].value)
+    r += ser_compact_size(len(spec[1]))
+    r += spec[1]
+    r += struct.pack('>H', spec[2])
+    return r
+
+def process_nodes(g, f, structname):
+    g.write('static const uint8_t %s[] = {\n' % structname)
     for line in f:
         comment = line.find('#')
         if comment != -1:
@@ -103,14 +127,12 @@ def process_nodes(g, f, structname, defaultport):
         line = line.strip()
         if not line:
             continue
-        if not first:
-            g.write(',\n')
-        first = False
 
-        (host,port) = parse_spec(line, defaultport)
-        hoststr = ','.join(('0x%02x' % b) for b in host)
-        g.write('    {{%s}, %i}' % (hoststr, port))
-    g.write('\n};\n')
+        spec = parse_spec(line)
+        blob = bip155_serialize(spec)
+        hoststr = ','.join(('0x%02x' % b) for b in blob)
+        g.write(f'    {hoststr},\n')
+    g.write('};\n')
 
 def main():
     if len(sys.argv)<2:
@@ -124,14 +146,13 @@ def main():
     g.write(' * List of fixed seed nodes for the bitcoin network\n')
     g.write(' * AUTOGENERATED by contrib/seeds/generate-seeds.py\n')
     g.write(' *\n')
-    g.write(' * Each line contains a 16-byte IPv6 address and a port.\n')
-    g.write(' * IPv4 as well as onion addresses are wrapped inside an IPv6 address accordingly.\n')
+    g.write(' * Each line contains a BIP155 serialized (networkID, addr, port) tuple.\n')
     g.write(' */\n')
     with open(os.path.join(indir,'nodes_main.txt'), 'r', encoding="utf8") as f:
-        process_nodes(g, f, 'pnSeed6_main', 8333)
+        process_nodes(g, f, 'chainparams_seed_main')
     g.write('\n')
     with open(os.path.join(indir,'nodes_test.txt'), 'r', encoding="utf8") as f:
-        process_nodes(g, f, 'pnSeed6_test', 18333)
+        process_nodes(g, f, 'chainparams_seed_test')
     g.write('#endif // BITCOIN_CHAINPARAMSSEEDS_H\n')
 
 if __name__ == '__main__':

contrib/seeds/nodes_main.txt

@@ -1162,3 +1162,29 @@ zuytrfevzjcpizli.onion:8333
 zvq6dpt3i2ofdp3g.onion:8333
 zwwm6ga7u2hqe2sd.onion:8333
 zyqb4lenfspntj5m.onion:8333
+
+# manually added 2021-03 for minimal torv3 bootstrap support
+2g5qfdkn2vvcbqhzcyvyiitg4ceukybxklraxjnu7atlhd22gdwywaid.onion:8333
+2jmtxvyup3ijr7u6uvu7ijtnojx4g5wodvaedivbv74w4vzntxbrhvad.onion:8333
+37m62wn7dz3uqpathpc4qfmgrbupachj52nt3jbtbjugpbu54kbud7yd.onion:8333
+5g72ppm3krkorsfopcm2bi7wlv4ohhs4u4mlseymasn7g7zhdcyjpfid.onion:8333
+7cgwjuwi5ehvcay4tazy7ya6463bndjk6xzrttw5t3xbpq4p22q6fyid.onion:8333
+7pyrpvqdhmayxggpcyqn5l3m5vqkw3qubnmgwlpya2mdo6x7pih7r7id.onion:8333
+b64xcbleqmwgq2u46bh4hegnlrzzvxntyzbmucn3zt7cssm7y4ubv3id.onion:8333
+ejxefzf5fpst4mg2rib7grksvscl7p6fvjp6agzgfc2yglxnjtxc3aid.onion:8333
+fjdyxicpm4o42xmedlwl3uvk5gmqdfs5j37wir52327vncjzvtpfv7yd.onion:8333
+fpz6r5ppsakkwypjcglz6gcnwt7ytfhxskkfhzu62tnylcknh3eq6pad.onion:8333
+fzhn4uoxfbfss7h7d6ffbn266ca432ekbbzvqtsdd55ylgxn4jucm5qd.onion:8333
+gxo5anvfnffnftfy5frkgvplq3rpga2ie3tcblo2vl754fvnhgorn5yd.onion:8333
+ifdu5qvbofrt4ekui2iyb3kbcyzcsglazhx2hn4wfskkrx2v24qxriid.onion:8333
+itz3oxsihs62muvknc237xabl5f6w6rfznfhbpayrslv2j2ubels47yd.onion:8333
+lrjh6fywjqttmlifuemq3puhvmshxzzyhoqx7uoufali57eypuenzzid.onion:8333
+m7cbpjolo662uel7rpaid46as2otcj44vvwg3gccodnvaeuwbm3anbyd.onion:8333
+opnyfyeiibe5qo5a3wbxzbb4xdiagc32bbce46owmertdknta5mi7uyd.onion:8333
+owjsdxmzla6d7lrwkbmetywqym5cyswpihciesfl5qdv2vrmwsgy4uqd.onion:8333
+q7kgmd7n7h27ds4fg7wocgniuqb3oe2zxp4nfe4skd5da6wyipibqzqd.onion:8333
+rp7k2go3s5lyj3fnj6zn62ktarlrsft2ohlsxkyd7v3e3idqyptvread.onion:8333
+sys54sv4xv3hn3sdiv3oadmzqpgyhd4u4xphv4xqk64ckvaxzm57a7yd.onion:8333
+tddeij4qigtjr6jfnrmq6btnirmq5msgwcsdpcdjr7atftm7cxlqztid.onion:8333
+vi5bnbxkleeqi6hfccjochnn65lcxlfqs4uwgmhudph554zibiusqnad.onion:8333
+xqt25cobm5zqucac3634zfght72he6u3eagfyej5ellbhcdgos7t2had.onion:8333

contrib/seeds/nodes_test.txt

@@ -1,11 +1,11 @@
 # List of fixed seed nodes for testnet
 
 # Onion nodes
-thfsmmn2jbitcoin.onion
-it2pj4f7657g3rhi.onion
-nkf5e6b7pl4jfd4a.onion
-4zhkir2ofl7orfom.onion
-t6xj6wilh4ytvcs7.onion
-i6y6ivorwakd7nw3.onion
-ubqj4rsu3nqtxmtp.onion
+thfsmmn2jbitcoin.onion:18333
+it2pj4f7657g3rhi.onion:18333
+nkf5e6b7pl4jfd4a.onion:18333
+4zhkir2ofl7orfom.onion:18333
+t6xj6wilh4ytvcs7.onion:18333
+i6y6ivorwakd7nw3.onion:18333
+ubqj4rsu3nqtxmtp.onion:18333
 

src/chainparams.cpp

@@ -134,7 +134,7 @@ public:
 
         bech32_hrp = "bc";
 
-        vFixedSeeds = std::vector<SeedSpec6>(std::begin(pnSeed6_main), std::end(pnSeed6_main));
+        vFixedSeeds = std::vector<uint8_t>(std::begin(chainparams_seed_main), std::end(chainparams_seed_main));
 
         fDefaultConsistencyChecks = false;
         fRequireStandard = true;
@@ -239,7 +239,7 @@ public:
 
         bech32_hrp = "tb";
 
-        vFixedSeeds = std::vector<SeedSpec6>(std::begin(pnSeed6_test), std::end(pnSeed6_test));
+        vFixedSeeds = std::vector<uint8_t>(std::begin(chainparams_seed_test), std::end(chainparams_seed_test));
 
         fDefaultConsistencyChecks = false;
         fRequireStandard = false;

src/chainparams.h

@@ -14,11 +14,6 @@
 #include <memory>
 #include <vector>
 
-struct SeedSpec6 {
-    uint8_t addr[16];
-    uint16_t port;
-};
-
 typedef std::map<int, uint256> MapCheckpoints;
 
 struct CCheckpointData {
@@ -105,7 +100,7 @@ public:
     const std::vector<std::string>& DNSSeeds() const { return vSeeds; }
     const std::vector<unsigned char>& Base58Prefix(Base58Type type) const { return base58Prefixes[type]; }
     const std::string& Bech32HRP() const { return bech32_hrp; }
-    const std::vector<SeedSpec6>& FixedSeeds() const { return vFixedSeeds; }
+    const std::vector<uint8_t>& FixedSeeds() const { return vFixedSeeds; }
     const CCheckpointData& Checkpoints() const { return checkpointData; }
 
     //! Get allowed assumeutxo configuration.
@@ -127,7 +122,7 @@ protected:
     std::string bech32_hrp;
     std::string strNetworkID;
     CBlock genesis;
-    std::vector<SeedSpec6> vFixedSeeds;
+    std::vector<uint8_t> vFixedSeeds;
     bool fDefaultConsistencyChecks;
     bool fRequireStandard;
     bool m_is_test_chain;

src/net.cpp

@@ -145,8 +145,8 @@ bool GetLocal(CService& addr, const CNetAddr *paddrPeer)
     return nBestScore >= 0;
 }
 
-//! Convert the pnSeed6 array into usable address objects.
-static std::vector<CAddress> convertSeed6(const std::vector<SeedSpec6> &vSeedsIn)
+//! Convert the serialized seeds into usable address objects.
+static std::vector<CAddress> ConvertSeeds(const std::vector<uint8_t> &vSeedsIn)
 {
     // It'll only connect to one or two seed nodes because once it connects,
     // it'll get a pile of addresses with newer timestamps.
@@ -154,13 +154,14 @@ static std::vector<CAddress> convertSeed6(const std::vector<SeedSpec6> &vSeedsIn
     // weeks ago.
     const int64_t nOneWeek = 7*24*60*60;
     std::vector<CAddress> vSeedsOut;
-    vSeedsOut.reserve(vSeedsIn.size());
     FastRandomContext rng;
-    for (const auto& seed_in : vSeedsIn) {
-        struct in6_addr ip;
-        memcpy(&ip, seed_in.addr, sizeof(ip));
-        CAddress addr(CService(ip, seed_in.port), GetDesirableServiceFlags(NODE_NONE));
+    CDataStream s(vSeedsIn, SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT);
+    while (!s.eof()) {
+        CService endpoint;
+        s >> endpoint;
+        CAddress addr{endpoint, GetDesirableServiceFlags(NODE_NONE)};
         addr.nTime = GetTime() - rng.randrange(nOneWeek) - nOneWeek;
+        LogPrint(BCLog::NET, "Added hardcoded seed: %s\n", addr.ToString());
         vSeedsOut.push_back(addr);
     }
     return vSeedsOut;
@@ -1847,7 +1848,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect)
             if (add_fixed_seeds_now) {
                 CNetAddr local;
                 local.SetInternal("fixedseeds");
-                addrman.Add(convertSeed6(Params().FixedSeeds()), local);
+                addrman.Add(ConvertSeeds(Params().FixedSeeds()), local);
                 add_fixed_seeds = false;
             }
         }