Merge #20339: ci: Run more ci configs on cirrus

77777c8b5e ci: Run windows ci config on cirrus (MarcoFalke)
3333d6942e ci: Run macos ci config on cirrus (MarcoFalke)
fa8b1114e6 ci: Run arm ci config on cirrus (MarcoFalke)
fa0795f54d ci: Replace TRAVIS_OS_NAME with CI_OS_NAME (MarcoFalke)
fafce1a13a ci: Move documentation to correct config file (MarcoFalke)

Pull request description:

ACKs for top commit:
  hebasto:
    ACK 77777c8b5e

Tree-SHA512: 581d3bdb2c3e3da20bd8492c6b23d90f3b4f7f9300ade384667e41de9fbbe42bbcef26dd99328839dc0b49be6e303c4c6d1b66ceb5d8009800c52c97685f9080

.cirrus.yml

@@ -8,7 +8,7 @@ container:
   memory: 8G  # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers
   kvm: true  # Use kvm to avoid spurious CI failures in the default virtualization cluster, see https://github.com/bitcoin/bitcoin/issues/20093
 env:
-  PACKAGE_MANAGER_INSTALL : "apt-get update && apt-get install -y"
+  PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y"
   MAKEJOBS: "-j4"
   DANGER_RUN_CI_ON_HOST: "1"  # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system
   TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
@@ -53,6 +53,22 @@ global_task_template: &GLOBAL_TASK_TEMPLATE
 #  install_script:
 #    - choco install python --version=3.7.7 -y
 
+task:
+  name: 'ARM  [GOAL: install]  [buster]  [unit tests, no functional tests]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: debian:buster
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_arm.sh"
+
+task:
+  name: 'Win64  [GOAL: deploy]  [unit tests, no gui, no boost::process, no functional tests]'
+  << : *GLOBAL_TASK_TEMPLATE
+  container:
+    image: ubuntu:bionic
+  env:
+    FILE_ENV: "./ci/test/00_setup_env_win64.sh"
+
 task:
   name: 'x86_64 Linux  [GOAL: install]  [bionic]  [C++17, previous releases, uses qt5 dev package and some depends packages] [unsigned char]'
   << : *GLOBAL_TASK_TEMPLATE
@@ -119,3 +135,17 @@ task:
     image: ubuntu:bionic
   env:
     FILE_ENV: "./ci/test/00_setup_env_mac.sh"
+
+task:
+  name: 'macOS 10.14 native [GOAL: install] [GUI] [no depends]'
+  macos_brew_addon_script:
+    - brew install boost libevent berkeley-db4 qt miniupnpc ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt
+  << : *GLOBAL_TASK_TEMPLATE
+  osx_instance:
+    # Use latest image, but hardcode version to avoid silent upgrades (and breaks)
+    image: catalina-xcode-12.1  # https://cirrus-ci.org/guide/macOS
+  env:
+    DANGER_RUN_CI_ON_HOST: "true"
+    CI_USE_APT_INSTALL: "no"
+    PACKAGE_MANAGER_INSTALL: "echo"  # Nothing to do
+    FILE_ENV: "./ci/test/00_setup_env_mac_host.sh"

.travis.yml

@@ -65,18 +65,6 @@ jobs:
       script:
         - set -o errexit; source ./ci/lint/06_script.sh
 
-    - stage: test
-      name: 'ARM  [GOAL: install]  [buster]  [unit tests, no functional tests]'
-      arch: arm64  # Can disable QEMU_USER_CMD and run the tests natively without qemu
-      env: >-
-        FILE_ENV="./ci/test/00_setup_env_arm.sh"
-        QEMU_USER_CMD=""
-
-    - stage: test
-      name: 'Win64  [GOAL: deploy]  [unit tests, no gui, no boost::process, no functional tests]'
-      env: >-
-        FILE_ENV="./ci/test/00_setup_env_win64.sh"
-
     - stage: test
       name: '32-bit + dash  [GOAL: install]  [CentOS 7]  [gui]'
       env: >-
@@ -86,24 +74,3 @@ jobs:
       name: 'x86_64 Linux  [GOAL: install]  [xenial]  [no wallet]'
       env: >-
         FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh"
-
-    - stage: test
-      name: 'macOS 10.14 native [GOAL: install] [GUI] [no depends]'
-      os: osx
-      # Use the most recent version:
-      # Xcode 11.3.1, macOS 10.14, SDK 10.15
-      # https://docs.travis-ci.com/user/reference/osx/#macos-version
-      osx_image: xcode11.3
-      addons:
-        homebrew:
-          packages:
-          - berkeley-db4
-          - miniupnpc
-          - qrencode
-          - sqlite
-          - ccache
-          - zeromq
-      env: >-
-        DANGER_RUN_CI_ON_HOST=true
-        CI_USE_APT_INSTALL=no
-        FILE_ENV="./ci/test/00_setup_env_mac_host.sh"

ci/test/00_setup_env_mac.sh

@@ -7,6 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_macos_cross
+export DOCKER_NAME_TAG=ubuntu:18.04  # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well)
 export HOST=x86_64-apple-darwin16
 export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools"
 export XCODE_VERSION=11.3.1

ci/test/00_setup_env_mac_host.sh

@@ -7,16 +7,12 @@
 export LC_ALL=C.UTF-8
 
 export HOST=x86_64-apple-darwin16
-export DOCKER_NAME_TAG=ubuntu:18.04  # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well)
 export PIP_PACKAGES="zmq"
 export GOAL="install"
 export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror --with-boost-process"
+export CI_OS_NAME="macos"
 export NO_DEPENDS=1
 export OSX_SDK=""
 export CCACHE_SIZE=300M
 
 export RUN_SECURITY_TESTS="true"
-if [ "$TRAVIS_REPO_SLUG" != "bitcoin/bitcoin" ]; then
-  export RUN_FUNCTIONAL_TESTS="false"
-  export EXPECTED_TESTS_DURATION_IN_SECONDS=200
-fi

ci/test/00_setup_env_win64.sh

@@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8
 export CONTAINER_NAME=ci_win64
 export DOCKER_NAME_TAG=ubuntu:18.04  # Check that bionic can cross-compile to win64 (bionic is used in the gitian build as well)
 export HOST=x86_64-w64-mingw32
-export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64"
+export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 file"
 export RUN_FUNCTIONAL_TESTS=false
 export RUN_SECURITY_TESTS="true"
 export GOAL="deploy"

ci/test/04_install.sh

@@ -13,8 +13,8 @@ if [[ $QEMU_USER_CMD == qemu-s390* ]]; then
   export LC_ALL=C
 fi
 
-if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+if [ "$CI_OS_NAME" == "macos" ]; then
-  ${CI_RETRY_EXE} pip3 install $PIP_PACKAGES
+  IN_GETOPT_BIN="/usr/local/opt/gnu-getopt/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
 fi
 
 # Create folders that are mounted into the docker
@@ -26,9 +26,7 @@ export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/l
 export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan"
 export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
 env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env
-if [[ $HOST = *-mingw32 ]]; then
+if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
-  DOCKER_ADMIN="--cap-add SYS_ADMIN"
-elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
   DOCKER_ADMIN="--cap-add SYS_PTRACE"
 fi
 
@@ -69,16 +67,16 @@ elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
   ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES
 fi
 
-if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+if [ "$CI_OS_NAME" == "macos" ]; then
   top -l 1 -s 0 | awk ' /PhysMem/ {print}'
   echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
 else
   DOCKER_EXEC free -m -h
   DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\)
   DOCKER_EXEC echo $(lscpu | grep Endian)
-  DOCKER_EXEC echo "Free disk space:"
-  DOCKER_EXEC df -h
 fi
+DOCKER_EXEC echo "Free disk space:"
+DOCKER_EXEC df -h
 
 if [ ! -d ${DIR_QA_ASSETS} ]; then
   DOCKER_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}

ci/test/05_before_script.sh

@@ -7,7 +7,7 @@
 export LC_ALL=C.UTF-8
 
 # Make sure default datadir does not exist and is never read by creating a dummy file
-if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+if [ "$CI_OS_NAME" == "macos" ]; then
   echo > $HOME/Library/Application\ Support/Bitcoin
 else
   DOCKER_EXEC echo \> \$HOME/.bitcoin

ci/test/06_script_b.sh

@@ -6,6 +6,15 @@
 
 export LC_ALL=C.UTF-8
 
+if [[ $HOST = *-mingw32 ]]; then
+  BEGIN_FOLD wrap-wine
+  # Generate all binaries, so that they can be wrapped
+  DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1
+  DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1
+  DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh"
+  END_FOLD
+fi
+
 if [ -n "$QEMU_USER_CMD" ]; then
   BEGIN_FOLD wrap-qemu
   # Generate all binaries, so that they can be wrapped

ci/test/wrap-wine.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}.exe; do
+    # shellcheck disable=SC2044
+    for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename $b_name)"); do
+      if (file "$b" | grep "Windows"); then
+        echo "Wrap $b ..."
+        mv "$b" "${b}_orig"
+        echo '#!/usr/bin/env bash' > "$b"
+        echo "wine64 \"${b}_orig\" \"\$@\"" >> "$b"
+        chmod +x "$b"
+      fi
+    done
+done