From 4fcab2957deafc9f66e5b73a8b19c5dd71125d56 Mon Sep 17 00:00:00 2001 From: GarmashAlex Date: Mon, 28 Apr 2025 16:49:47 +0300 Subject: [PATCH 1/9] test: Add Windows binaries support to previous releases downloader --- test/get_previous_releases.py | 38 +++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index 1f8d92c980d..d39e5702fc7 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -106,6 +106,12 @@ SHA256_SUMS = { "6ee1a520b638132a16725020146abea045db418ce91c02493f02f541cd53062a": {"tag": "v28.0", "tarball": "bitcoin-28.0-riscv64-linux-gnu.tar.gz"}, "77e931bbaaf47771a10c376230bf53223f5380864bad3568efc7f4d02e40a0f7": {"tag": "v28.0", "tarball": "bitcoin-28.0-x86_64-apple-darwin.tar.gz"}, "7fe294b02b25b51acb8e8e0a0eb5af6bbafa7cd0c5b0e5fcbb61263104a82fbc": {"tag": "v28.0", "tarball": "bitcoin-28.0-x86_64-linux-gnu.tar.gz"}, + + "bff531650dcf859c27d8428dc5f98f3f93d9b6d54e4c1401e0ea9651f1edd7a3": {"tag": "v22.0", "tarball": "bitcoin-22.0-win64.zip"}, + "e16fdbdc4ee953969df1ce8f82380e9c61658b1a64801bd21e2e87063e93bc6a": {"tag": "v23.0", "tarball": "bitcoin-23.0-win64.zip"}, + "97de6e2a4d91531c057537e83bb7e72b1cf9ab777601eb481f3f4a6d9e7b9c67": {"tag": "v24.0.1", "tarball": "bitcoin-24.0.1-win64.zip"}, + "c0d22e9d37d0238215676af1c4e358c8af3a43fe7a25c57499b4c66ca80381da": {"tag": "v25.0", "tarball": "bitcoin-25.0-win64.zip"}, + "f2974a7df505cff14ca92dd7a23ba7e47f1b97ae7e7a12a6fc2f5f5c0a66ca10": {"tag": "v28.0", "tarball": "bitcoin-28.0-win64.zip"}, } @@ -134,8 +140,13 @@ def download_binary(tag, args) -> int: platform = args.platform if tag < "v23" and platform in ["x86_64-apple-darwin", "arm64-apple-darwin"]: platform = "osx64" + if tag < "v23" and platform == "win64": + platform = "win64" # Windows platform names have been consistent tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format( tag=tag[1:], platform=platform) + if platform == "win64" and tag >= "v22.0": + tarball = 'bitcoin-{tag}-{platform}.zip'.format( + tag=tag[1:], platform=platform) tarballUrl = 'https://bitcoincore.org/{bin_path}/{tarball}'.format( bin_path=bin_path, tarball=tarball) @@ -160,12 +171,25 @@ def download_binary(tag, args) -> int: print("Checksum matched") # Extract tarball - ret = subprocess.run(['tar', '-zxf', tarball, '-C', tag, - '--strip-components=1', - 'bitcoin-{tag}'.format(tag=tag[1:])]).returncode - if ret != 0: - print(f"Failed to extract the {tag} tarball") - return ret + if platform == "win64" and tag >= "v22.0": + # Handle zip files for Windows + import zipfile + with zipfile.ZipFile(tarball, 'r') as zip_ref: + zip_ref.extractall(tag) + # Rename the directory to match expected structure + extracted_dir = Path(tag) / f"bitcoin-{tag[1:]}" + if extracted_dir.exists(): + for item in extracted_dir.iterdir(): + shutil.move(str(item), str(Path(tag) / item.name)) + extracted_dir.rmdir() + else: + # Standard tar extraction + ret = subprocess.run(['tar', '-zxf', tarball, '-C', tag, + '--strip-components=1', + 'bitcoin-{tag}'.format(tag=tag[1:])]).returncode + if ret != 0: + print(f"Failed to extract the {tag} tarball") + return ret Path(tarball).unlink() @@ -258,6 +282,8 @@ def check_host(args) -> int: 'x86_64-*-linux*': 'x86_64-linux-gnu', 'x86_64-apple-darwin*': 'x86_64-apple-darwin', 'aarch64-apple-darwin*': 'arm64-apple-darwin', + 'x86_64-w64-mingw32': 'win64', + 'x86_64-*-win*': 'win64', } args.platform = '' for pattern, target in platforms.items(): From 819140ef810a41a215516fa1835be7fd73066a6b Mon Sep 17 00:00:00 2001 From: GarmashAlex Date: Mon, 28 Apr 2025 16:50:29 +0300 Subject: [PATCH 2/9] test: Fix wallet_migration.py for Windows platform --- test/functional/wallet_migration.py | 34 +++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/test/functional/wallet_migration.py b/test/functional/wallet_migration.py index 2f5c3b1548c..4b178c20e7f 100755 --- a/test/functional/wallet_migration.py +++ b/test/functional/wallet_migration.py @@ -8,6 +8,8 @@ import random import shutil import struct import time +import platform +import os from test_framework.address import ( key_to_p2pkh, @@ -104,10 +106,38 @@ class WalletMigrationTest(BitcoinTestFramework): assert_equal(self.old_node.get_wallet_rpc(wallet_name).getwalletinfo()["descriptors"], False) # Now unload so we can copy it to the master node for the migration test self.old_node.unloadwallet(wallet_name) + + # Handle different path behavior on Windows vs. other platforms + is_windows = platform.system() == 'Windows' + if wallet_name == "": - shutil.copyfile(self.old_node.wallets_path / "wallet.dat", self.master_node.wallets_path / "wallet.dat") + src_path = self.old_node.wallets_path / "wallet.dat" + dst_path = self.master_node.wallets_path / "wallet.dat" + # On Windows, we need to close all open file handles before copying + if is_windows: + import gc + gc.collect() # Try to release any file handles + shutil.copyfile(src_path, dst_path) else: - shutil.copytree(self.old_node.wallets_path / wallet_name, self.master_node.wallets_path / wallet_name) + src_path = self.old_node.wallets_path / wallet_name + dst_path = self.master_node.wallets_path / wallet_name + # On Windows, we need to close all open file handles before copying + if is_windows: + import gc + gc.collect() # Try to release any file handles + + # Create the directory first if it doesn't exist + os.makedirs(dst_path, exist_ok=True) + + # Copy the files individually + for item in os.listdir(src_path): + s = os.path.join(src_path, item) + d = os.path.join(dst_path, item) + if os.path.isfile(s): + shutil.copy2(s, d) + else: + shutil.copytree(s, d, dirs_exist_ok=True) + # Migrate, checking that rescan does not occur with self.master_node.assert_debug_log(expected_msgs=[], unexpected_msgs=["Rescanning"]): migrate_info = self.master_node.migratewallet(wallet_name=wallet_name, **kwargs) From 60f9f1f0239be0511367b27a2514969a16e09eae Mon Sep 17 00:00:00 2001 From: GarmashAlex Date: Mon, 28 Apr 2025 16:51:00 +0300 Subject: [PATCH 3/9] ci: Enable previous releases tests on Windows CI --- .github/workflows/ci.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d8fdb4a16e..b0c48fa4da9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,6 +168,7 @@ jobs: env: PYTHONUTF8: 1 TEST_RUNNER_TIMEOUT_FACTOR: 40 + DOWNLOAD_PREVIOUS_RELEASES: "true" strategy: fail-fast: false @@ -334,6 +335,8 @@ jobs: env: PYTHONUTF8: 1 TEST_RUNNER_TIMEOUT_FACTOR: 40 + DOWNLOAD_PREVIOUS_RELEASES: "true" + PREVIOUS_RELEASES_DIR: ${{ github.workspace }}/releases steps: - name: Checkout @@ -372,10 +375,24 @@ jobs: - name: Run rpcauth test run: py -3 test/util/rpcauth-test.py + - name: Download previous releases + if: env.DOWNLOAD_PREVIOUS_RELEASES == 'true' + shell: bash + run: | + mkdir -p ${{ env.PREVIOUS_RELEASES_DIR }} + python test/get_previous_releases.py -b -t "${{ env.PREVIOUS_RELEASES_DIR }}" + + - name: Run wallet_migration.py test + if: env.DOWNLOAD_PREVIOUS_RELEASES == 'true' + env: + PYTHONIOENCODING: utf-8 + run: py -3 test/functional/wallet_migration.py --previous-releases + - name: Run functional tests env: # TODO: Fix the excluded test and re-enable it. - EXCLUDE: '--exclude wallet_multiwallet.py' + EXCLUDE: '--exclude wallet_multiwallet.py --exclude wallet_migration.py' + PYTHONIOENCODING: utf-8 TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }} run: py -3 test/functional/test_runner.py --jobs $NUMBER_OF_PROCESSORS --ci --quiet --tmpdirprefix="$RUNNER_TEMP" --combinedlogslen=99999999 --timeout-factor=$TEST_RUNNER_TIMEOUT_FACTOR $EXCLUDE $TEST_RUNNER_EXTRA From c866e5e7258aafb2bf647a4acd1d781eeaf82c31 Mon Sep 17 00:00:00 2001 From: GarmashAlex Date: Mon, 28 Apr 2025 17:33:42 +0300 Subject: [PATCH 4/9] remove the exclusion --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0c48fa4da9..55e73354198 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -391,7 +391,7 @@ jobs: - name: Run functional tests env: # TODO: Fix the excluded test and re-enable it. - EXCLUDE: '--exclude wallet_multiwallet.py --exclude wallet_migration.py' + EXCLUDE: '--exclude wallet_multiwallet.py' PYTHONIOENCODING: utf-8 TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }} run: py -3 test/functional/test_runner.py --jobs $NUMBER_OF_PROCESSORS --ci --quiet --tmpdirprefix="$RUNNER_TEMP" --combinedlogslen=99999999 --timeout-factor=$TEST_RUNNER_TIMEOUT_FACTOR $EXCLUDE $TEST_RUNNER_EXTRA From 7e6a0fe7e659aec2844cb03177dacd06d9aa869c Mon Sep 17 00:00:00 2001 From: GarmashAlex Date: Mon, 28 Apr 2025 17:38:27 +0300 Subject: [PATCH 5/9] delete unnecessary code --- test/functional/wallet_migration.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/functional/wallet_migration.py b/test/functional/wallet_migration.py index 4b178c20e7f..9755b25692a 100755 --- a/test/functional/wallet_migration.py +++ b/test/functional/wallet_migration.py @@ -113,18 +113,10 @@ class WalletMigrationTest(BitcoinTestFramework): if wallet_name == "": src_path = self.old_node.wallets_path / "wallet.dat" dst_path = self.master_node.wallets_path / "wallet.dat" - # On Windows, we need to close all open file handles before copying - if is_windows: - import gc - gc.collect() # Try to release any file handles shutil.copyfile(src_path, dst_path) else: src_path = self.old_node.wallets_path / wallet_name dst_path = self.master_node.wallets_path / wallet_name - # On Windows, we need to close all open file handles before copying - if is_windows: - import gc - gc.collect() # Try to release any file handles # Create the directory first if it doesn't exist os.makedirs(dst_path, exist_ok=True) From 6b3920f6ab05781789ce54b95311c03ca7d6e72a Mon Sep 17 00:00:00 2001 From: GarmashAlex Date: Mon, 28 Apr 2025 17:40:43 +0300 Subject: [PATCH 6/9] delete useless code --- test/get_previous_releases.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index d39e5702fc7..4df4a7e1b3d 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -140,8 +140,6 @@ def download_binary(tag, args) -> int: platform = args.platform if tag < "v23" and platform in ["x86_64-apple-darwin", "arm64-apple-darwin"]: platform = "osx64" - if tag < "v23" and platform == "win64": - platform = "win64" # Windows platform names have been consistent tarball = 'bitcoin-{tag}-{platform}.tar.gz'.format( tag=tag[1:], platform=platform) if platform == "win64" and tag >= "v22.0": From aa31d6ba223810435f5a043c66075a74dd63d33e Mon Sep 17 00:00:00 2001 From: GarmashAlex Date: Mon, 28 Apr 2025 17:43:06 +0300 Subject: [PATCH 7/9] correct lint --- test/functional/wallet_migration.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/functional/wallet_migration.py b/test/functional/wallet_migration.py index 9755b25692a..9e84b43730f 100755 --- a/test/functional/wallet_migration.py +++ b/test/functional/wallet_migration.py @@ -107,9 +107,6 @@ class WalletMigrationTest(BitcoinTestFramework): # Now unload so we can copy it to the master node for the migration test self.old_node.unloadwallet(wallet_name) - # Handle different path behavior on Windows vs. other platforms - is_windows = platform.system() == 'Windows' - if wallet_name == "": src_path = self.old_node.wallets_path / "wallet.dat" dst_path = self.master_node.wallets_path / "wallet.dat" From 7987f07e9321b97bd7cc6033ed0e95f226a8150b Mon Sep 17 00:00:00 2001 From: GarmashAlex Date: Mon, 28 Apr 2025 20:29:02 +0300 Subject: [PATCH 8/9] Update get_previous_releases.py --- test/get_previous_releases.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index 4df4a7e1b3d..352712c6d43 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -270,8 +270,12 @@ def build_release(tag, args) -> int: def check_host(args) -> int: - args.host = os.environ.get('HOST', subprocess.check_output( - './depends/config.guess').decode()) + # On Windows, config.guess script doesn't work, so we set host manually + if sys.platform == 'win32': + args.host = 'x86_64-w64-mingw32' + else: + args.host = os.environ.get('HOST', subprocess.check_output( + './depends/config.guess').decode()) if args.download_binary: platforms = { 'aarch64-*-linux*': 'aarch64-linux-gnu', From bcd1d6ba648e3d0bb0f6d85864d100eefc96485c Mon Sep 17 00:00:00 2001 From: GarmashAlex Date: Mon, 28 Apr 2025 20:29:21 +0300 Subject: [PATCH 9/9] Update wallet_migration.py --- test/functional/wallet_migration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/functional/wallet_migration.py b/test/functional/wallet_migration.py index 9e84b43730f..eec187ba348 100755 --- a/test/functional/wallet_migration.py +++ b/test/functional/wallet_migration.py @@ -8,7 +8,6 @@ import random import shutil import struct import time -import platform import os from test_framework.address import (