diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 021d5e15976..94149106ee2 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -64,7 +64,7 @@ export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out} # The folder for previous release binaries. # This folder exists only on the ci guest, and on the ci host as a volume. export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/prev_releases} -export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential pkg-config curl ca-certificates ccache python3 rsync git procps bison e2fsprogs cmake} +export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential pkg-config curl ca-certificates ccache python3 rsync git procps bison e2fsprogs cmake net-tools tcpdump} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"} diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 22657742d7c..d6bd9318704 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -10,7 +10,7 @@ export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_centos export CI_IMAGE_NAME_TAG="quay.io/centos/amd64:stream9" export STREAM_GCC_V="12" -export CI_BASE_PACKAGES="gcc-toolset-${STREAM_GCC_V}-gcc-c++ glibc-devel.x86_64 gcc-toolset-${STREAM_GCC_V}-libstdc++-devel.x86_64 glibc-devel.i686 gcc-toolset-${STREAM_GCC_V}-libstdc++-devel.i686 ccache make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison e2fsprogs cmake" +export CI_BASE_PACKAGES="gcc-toolset-${STREAM_GCC_V}-gcc-c++ glibc-devel.x86_64 gcc-toolset-${STREAM_GCC_V}-libstdc++-devel.x86_64 glibc-devel.i686 gcc-toolset-${STREAM_GCC_V}-libstdc++-devel.i686 ccache make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison e2fsprogs cmake net-tools tcpdump" export PIP_PACKAGES="pyzmq" export GOAL="install" export BITCOIN_CONFIG="-DWITH_ZMQ=ON -DBUILD_GUI=ON -DREDUCE_EXPORTS=ON" diff --git a/ci/test/00_setup_env_mac_native.sh b/ci/test/00_setup_env_mac_native.sh index e01a56895bf..317694e6f47 100755 --- a/ci/test/00_setup_env_mac_native.sh +++ b/ci/test/00_setup_env_mac_native.sh @@ -15,3 +15,5 @@ export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DREDUCE_EXPORTS=ON" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" +# Can't run tcpdump: tcpdump: en0: You don't have permission to capture on that device +export CI_TCPDUMP_OK_TO_FAIL=1 diff --git a/ci/test/00_setup_env_mac_native_fuzz.sh b/ci/test/00_setup_env_mac_native_fuzz.sh index 1a453a4353f..977358d9cf4 100755 --- a/ci/test/00_setup_env_mac_native_fuzz.sh +++ b/ci/test/00_setup_env_mac_native_fuzz.sh @@ -14,3 +14,5 @@ export OSX_SDK="" export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true +# Can't run tcpdump: tcpdump: en0: You don't have permission to capture on that device +export CI_TCPDUMP_OK_TO_FAIL=1 diff --git a/ci/test/02_run_container.sh b/ci/test/02_run_container.sh index ce01db325ce..e1539e8bef3 100755 --- a/ci/test/02_run_container.sh +++ b/ci/test/02_run_container.sh @@ -91,7 +91,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then # Append $USER to /tmp/env to support multi-user systems and $CONTAINER_NAME # to allow support starting multiple runs simultaneously by the same user. # shellcheck disable=SC2086 - CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE $CI_CONTAINER_CAP --rm --interactive --detach --tty \ + CI_CONTAINER_ID=$(docker run --cap-add LINUX_IMMUTABLE --cap-add NET_RAW $CI_CONTAINER_CAP --rm --interactive --detach --tty \ --mount "type=bind,src=$BASE_READ_ONLY_DIR,dst=$BASE_READ_ONLY_DIR,readonly" \ --mount "${CI_CCACHE_MOUNT}" \ --mount "${CI_DEPENDS_MOUNT}" \ diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index 6e77a8927c2..7b10f3f27d5 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -146,21 +146,74 @@ if [ "$RUN_CHECK_DEPS" = "true" ]; then "${BASE_ROOT_DIR}/contrib/devtools/check-deps.sh" . fi +function get_interfaces() +{ + set -o pipefail + ifconfig | awk -F ':| ' '/^[^[:space:]]/ { if (!match($1, /^lo/)) { print $1 } }' + set +o pipefail +} + +function tcpdump_file() +{ + echo "/tmp/tcpdump_$1_$2" +} + +function traffic_monitor_begin() +{ + test_name="$1" + for ifname in $(get_interfaces) ; do + tcpdump -nU -i "$ifname" -w "$(tcpdump_file "$test_name" "$ifname")" & + done +} + +function traffic_monitor_end() +{ + test_name="$1" + + for ifname in $(get_interfaces) ; do + f=$(tcpdump_file "$test_name" "$ifname") + if [ ! -e "$f" ] && [ "$CI_TCPDUMP_OK_TO_FAIL" = "1" ] ; then + # In some CI environments this script is not running as root and so the + # tcpdump errors and does not create $f. Skip silently those, but we + # need at least one where tcpdump can run and this is the ASAN one. So + # treat the absence of $f as an error only on the ASAN task. + continue + fi + # We are running as root and those files are created with owner:group = + # tcpdump:tcpdump and then `tcpdump -r` refuses to read them with an error + # "permission denied" if they are not owned by root:root. + chown root:root "$f" + out="$(tcpdump -n -r "$f" --direction=out tcp or udp)" + if [ -n "$out" ] ; then + echo "Error: outbound TCP or UDP packets on the non loopback interface generated during $test_name tests:" >&2 + tcpdump -n -r "$f" tcp or udp + exit 1 + fi + done +} + if [ "$RUN_UNIT_TESTS" = "true" ]; then + traffic_monitor_begin "unitparallel" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" CTEST_OUTPUT_ON_FAILURE=ON ctest --stop-on-failure "${MAKEJOBS}" --timeout $(( TEST_RUNNER_TIMEOUT_FACTOR * 60 )) + traffic_monitor_end "unitparallel" fi if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then + traffic_monitor_begin "unitsequential" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_OUTDIR}"/bin/test_bitcoin --catch_system_errors=no -l test_suite + traffic_monitor_end "unitsequential" fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then + traffic_monitor_begin "functional" # parses TEST_RUNNER_EXTRA as an array which allows for multiple arguments such as TEST_RUNNER_EXTRA='--exclude "rpc_bind.py --ipv6"' eval "TEST_RUNNER_EXTRA=($TEST_RUNNER_EXTRA)" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/functional/test_runner.py --ci "${MAKEJOBS}" --tmpdirprefix "${BASE_SCRATCH_DIR}"/test_runner/ --ansi --combinedlogslen=99999999 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA[@]}" --quiet --failfast + traffic_monitor_end "functional" fi if [ "${RUN_TIDY}" = "true" ]; then + traffic_monitor_begin "tidy" cmake -B /tidy-build -DLLVM_DIR=/usr/lib/llvm-"${TIDY_LLVM_V}"/cmake -DCMAKE_BUILD_TYPE=Release -S "${BASE_ROOT_DIR}"/contrib/devtools/bitcoin-tidy cmake --build /tidy-build "$MAKEJOBS" cmake --build /tidy-build --target bitcoin-tidy-tests "$MAKEJOBS" @@ -185,9 +238,12 @@ if [ "${RUN_TIDY}" = "true" ]; then cd "${BASE_ROOT_DIR}/src" python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out git --no-pager diff + traffic_monitor_end "tidy" fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then + traffic_monitor_begin "fuzz" # shellcheck disable=SC2086 LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} "${MAKEJOBS}" -l DEBUG "${DIR_FUZZ_IN}" --empty_min_time=60 + traffic_monitor_end "fuzz" fi diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index 31847b3fbfc..05eab79061f 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -16,6 +16,10 @@ from test_framework.test_node import ErrorMatch from test_framework import util +# Easily unreachable address. Attempts to connect to it will stay within the machine. +# Used to avoid non-loopback traffic or DNS queries. +DUMMY_PROXY_ARG = '-proxy=127.0.0.1:1' + class ConfArgsTest(BitcoinTestFramework): def add_options(self, parser): self.add_wallet_options(parser) @@ -227,8 +231,8 @@ class ConfArgsTest(BitcoinTestFramework): ) def test_log_buffer(self): - with self.nodes[0].assert_debug_log(expected_msgs=['Warning: parsed potentially confusing double-negative -connect=0\n']): - self.start_node(0, extra_args=['-noconnect=0']) + with self.nodes[0].assert_debug_log(expected_msgs=['Warning: parsed potentially confusing double-negative -listen=0\n']): + self.start_node(0, extra_args=['-nolisten=0']) self.stop_node(0) def test_args_log(self): @@ -259,6 +263,7 @@ class ConfArgsTest(BitcoinTestFramework): '-rpcpassword=', '-rpcuser=secret-rpcuser', '-torpassword=secret-torpassword', + DUMMY_PROXY_ARG, ]) self.stop_node(0) @@ -307,7 +312,7 @@ class ConfArgsTest(BitcoinTestFramework): ], timeout=10, ): - self.start_node(0, extra_args=['-dnsseed=1', '-fixedseeds=1', f'-mocktime={start}']) + self.start_node(0, extra_args=['-dnsseed=1', '-fixedseeds=1', f'-mocktime={start}', DUMMY_PROXY_ARG]) # Only regtest has no fixed seeds. To avoid connections to random # nodes, regtest is the only network where it is safe to enable @@ -355,7 +360,7 @@ class ConfArgsTest(BitcoinTestFramework): ], timeout=10, ): - self.start_node(0, extra_args=['-dnsseed=0', '-fixedseeds=1', '-addnode=fakenodeaddr', f'-mocktime={start}']) + self.start_node(0, extra_args=['-dnsseed=0', '-fixedseeds=1', '-addnode=fakenodeaddr', f'-mocktime={start}', DUMMY_PROXY_ARG]) with self.nodes[0].assert_debug_log(expected_msgs=[ "Adding fixed seeds as 60 seconds have passed and addrman is empty", ]): @@ -371,18 +376,18 @@ class ConfArgsTest(BitcoinTestFramework): # When -connect is supplied, expanding addrman via getaddr calls to ADDR_FETCH(-seednode) # nodes is irrelevant and -seednode is ignored. with self.nodes[0].assert_debug_log(expected_msgs=seednode_ignored): - self.start_node(0, extra_args=['-connect=fakeaddress1', '-seednode=fakeaddress2']) + self.start_node(0, extra_args=['-connect=fakeaddress1', '-seednode=fakeaddress2', DUMMY_PROXY_ARG]) # With -proxy, an ADDR_FETCH connection is made to a peer that the dns seed resolves to. # ADDR_FETCH connections are not used when -connect is used. with self.nodes[0].assert_debug_log(expected_msgs=dnsseed_ignored): - self.restart_node(0, extra_args=['-connect=fakeaddress1', '-dnsseed=1', '-proxy=1.2.3.4']) + self.restart_node(0, extra_args=['-connect=fakeaddress1', '-dnsseed=1', DUMMY_PROXY_ARG]) # If the user did not disable -dnsseed, but it was soft-disabled because they provided -connect, # they shouldn't see a warning about -dnsseed being ignored. with self.nodes[0].assert_debug_log(expected_msgs=addcon_thread_started, unexpected_msgs=dnsseed_ignored): - self.restart_node(0, extra_args=['-connect=fakeaddress1', '-proxy=1.2.3.4']) + self.restart_node(0, extra_args=['-connect=fakeaddress1', DUMMY_PROXY_ARG]) # We have to supply expected_msgs as it's a required argument # The expected_msg must be something we are confident will be logged after the unexpected_msg diff --git a/test/functional/p2p_dns_seeds.py b/test/functional/p2p_dns_seeds.py index a2d4ea110f7..831b5165678 100755 --- a/test/functional/p2p_dns_seeds.py +++ b/test/functional/p2p_dns_seeds.py @@ -10,11 +10,16 @@ from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework +# Easily unreachable address. Attempts to connect to it will stay within the machine. +# Used to avoid non-loopback traffic or DNS queries. +DUMMY_PROXY_ARG = '-proxy=127.0.0.1:1' + + class P2PDNSSeeds(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 - self.extra_args = [["-dnsseed=1"]] + self.extra_args = [["-dnsseed=1", DUMMY_PROXY_ARG]] def run_test(self): self.init_arg_tests() @@ -29,11 +34,11 @@ class P2PDNSSeeds(BitcoinTestFramework): self.log.info("Check that setting -connect disables -dnsseed by default") self.nodes[0].stop_node() with self.nodes[0].assert_debug_log(expected_msgs=["DNS seeding disabled"]): - self.start_node(0, [f"-connect={fakeaddr}"]) + self.start_node(0, extra_args=[f"-connect={fakeaddr}", DUMMY_PROXY_ARG]) self.log.info("Check that running -connect and -dnsseed means DNS logic runs.") with self.nodes[0].assert_debug_log(expected_msgs=["Loading addresses from DNS seed"], timeout=12): - self.restart_node(0, [f"-connect={fakeaddr}", "-dnsseed=1"]) + self.restart_node(0, extra_args=[f"-connect={fakeaddr}", "-dnsseed=1", DUMMY_PROXY_ARG]) self.log.info("Check that running -forcednsseed and -dnsseed=0 throws an error.") self.nodes[0].stop_node() @@ -88,7 +93,7 @@ class P2PDNSSeeds(BitcoinTestFramework): with self.nodes[0].assert_debug_log(expected_msgs=["Loading addresses from DNS seed"], timeout=12): # -dnsseed defaults to 1 in bitcoind, but 0 in the test framework, # so pass it explicitly here - self.restart_node(0, ["-forcednsseed", "-dnsseed=1"]) + self.restart_node(0, ["-forcednsseed", "-dnsseed=1", DUMMY_PROXY_ARG]) # Restore default for subsequent tests self.restart_node(0) diff --git a/test/functional/p2p_seednode.py b/test/functional/p2p_seednode.py index 5a9fca3278a..01ee72ab52d 100755 --- a/test/functional/p2p_seednode.py +++ b/test/functional/p2p_seednode.py @@ -17,22 +17,24 @@ ADD_NEXT_SEEDNODE = 10 class P2PSeedNodes(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + # Specify a non-working proxy to make sure no actual connections to random IPs are attempted. + self.extra_args = [["-proxy=127.0.0.1:1"]] self.disable_autoconnect = False def test_no_seednode(self): self.log.info("Check that if no seednode is provided, the node proceeds as usual (without waiting)") with self.nodes[0].assert_debug_log(expected_msgs=[], unexpected_msgs=["Empty addrman, adding seednode", f"Couldn't connect to peers from addrman after {ADD_NEXT_SEEDNODE} seconds. Adding seednode"], timeout=ADD_NEXT_SEEDNODE): - self.restart_node(0) + self.restart_node(0, extra_args=self.nodes[0].extra_args) def test_seednode_empty_addrman(self): - seed_node = "0.0.0.1" + seed_node = "25.0.0.1" self.log.info("Check that the seednode is immediately added on bootstrap on an empty addrman") with self.nodes[0].assert_debug_log(expected_msgs=[f"Empty addrman, adding seednode ({seed_node}) to addrfetch"], timeout=ADD_NEXT_SEEDNODE): - self.restart_node(0, extra_args=[f'-seednode={seed_node}']) + self.restart_node(0, extra_args=self.nodes[0].extra_args + [f'-seednode={seed_node}']) def test_seednode_non_empty_addrman(self): self.log.info("Check that if addrman is non-empty, seednodes are queried with a delay") - seed_node = "0.0.0.2" + seed_node = "25.0.0.2" node = self.nodes[0] # Fill the addrman with unreachable nodes for i in range(10): @@ -40,9 +42,9 @@ class P2PSeedNodes(BitcoinTestFramework): port = 8333 + i node.addpeeraddress(ip, port) - # Restart the node so seednode is processed again. Specify a non-working proxy to make sure no actual connections to random IPs are attempted. + # Restart the node so seednode is processed again. with node.assert_debug_log(expected_msgs=["trying v1 connection"], timeout=ADD_NEXT_SEEDNODE): - self.restart_node(0, extra_args=[f'-seednode={seed_node}', '-proxy=127.0.0.1:1']) + self.restart_node(0, extra_args=self.nodes[0].extra_args + [f'-seednode={seed_node}']) with node.assert_debug_log(expected_msgs=[f"Couldn't connect to peers from addrman after {ADD_NEXT_SEEDNODE} seconds. Adding seednode ({seed_node}) to addrfetch"], unexpected_msgs=["Empty addrman, adding seednode"], timeout=ADD_NEXT_SEEDNODE * 1.5): node.setmocktime(int(time.time()) + ADD_NEXT_SEEDNODE + 1)