From 5d105fb8c3ffa39c3e716c3147ee74795b129113 Mon Sep 17 00:00:00 2001 From: Ryan Ofsky Date: Sat, 25 Jan 2025 13:33:44 -0500 Subject: [PATCH] depends: Switch libmultiprocess packages to use local git subtree With newly introduced libmultiprocess subtree, there's no need for depends system to download and track changes to the upstream repository. Note that adding the libmultiprocess subtree does not allow dropping libmultiprocess packages from the depends build, because libmultiprocess includes a code generation tool called mpgen, and in cross-compiled builds, bitcoin core's cmake build system doesn't have access to a native toolchain and can't build mpgen itself, so the depends system (or the native environment if not using depends) needs to supply it. --- depends/funcs.mk | 33 ++++++++++++++++++++++ depends/packages.md | 18 +++++++++++- depends/packages/libmultiprocess.mk | 1 + depends/packages/native_libmultiprocess.mk | 5 +--- 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/depends/funcs.mk b/depends/funcs.mk index f7a98b424ec..b499b6a78fe 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -39,9 +39,33 @@ define fetch_file $(call fetch_file_inner,$(1),$(FALLBACK_DOWNLOAD_PATH),$(3),$(4),$(5)))) endef +# Shell script to create a source tarball in $(1)_source from local directory +# $(1)_local_dir instead of downloading remote sources. Tarball is recreated if +# any paths in the local directory have a newer mtime, and checksum of the +# tarball is saved to $(1)_fetched and returned as output. +define fetch_local_dir_sha256 + if ! [ -f $($(1)_source) ] || [ -n "$$(find $($(1)_local_dir) -newer $($(1)_source) | head -n1)" ]; then \ + mkdir -p $(dir $($(1)_source)) && \ + $(build_TAR) -c -f $($(1)_source) -C $($(1)_local_dir) . && \ + rm -f $($(1)_fetched); \ + fi && \ + if ! [ -f $($(1)_fetched) ] || [ -n "$$(find $($(1)_source) -newer $($(1)_fetched))" ]; then \ + mkdir -p $(dir $($(1)_fetched)) && \ + cd $($(1)_source_dir) && \ + $(build_SHA256SUM) $($(1)_all_sources) > $($(1)_fetched); \ + fi && \ + cut -d" " -f1 $($(1)_fetched) +endef + define int_get_build_recipe_hash $(eval $(1)_patches_path?=$(PATCHES_PATH)/$(1)) $(eval $(1)_all_file_checksums:=$(shell $(build_SHA256SUM) $(meta_depends) packages/$(1).mk $(addprefix $($(1)_patches_path)/,$($(1)_patches)) | cut -d" " -f1)) +# If $(1)_local_dir is set, create a tarball of the local directory contents to +# use as the source of the package, and include a hash of the tarball in the +# package id, so if directory contents change, the package and packages +# depending on it will be rebuilt. +$(if $($(1)_local_dir),$(eval $(1)_sha256_hash:=$(shell $(call fetch_local_dir_sha256,$(1))))) +$(if $($(1)_local_dir),$(eval $(1)_all_file_checksums+=$($(1)_sha256_hash))) $(eval $(1)_recipe_hash:=$(shell echo -n "$($(1)_all_file_checksums)" | $(build_SHA256SUM) | cut -d" " -f1)) endef @@ -64,10 +88,19 @@ $(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).ta $(1)_build_log:=$(BASEDIR)/$(1)-$($(1)_version)-$($(1)_build_id).log endef +# Convert a string to a human-readable filename, replacing dot, slash, and space +# characters that could cause problems with dashes and collapsing them. +define int_friendly_file_name +$(subst $(null) $(null),-,$(strip $(subst ., ,$(subst /, ,$(1))))) +endef + define int_get_build_properties $(1)_build_subdir?=. $(1)_download_file?=$($(1)_file_name) $(1)_source_dir:=$(SOURCES_PATH) +# If $(1)_file_name is empty and $(1)_local_dir is nonempty, set file name to a +# .tar file with a friendly filename named after the directory path. +$(if $($(1)_file_name),,$(if $($(1)_local_dir),$(eval $(1)_file_name:=$(call int_friendly_file_name,$($(1)_local_dir)).tar))) $(1)_source:=$$($(1)_source_dir)/$($(1)_file_name) $(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version) $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ diff --git a/depends/packages.md b/depends/packages.md index 6b458f22dd4..f304f7e028a 100644 --- a/depends/packages.md +++ b/depends/packages.md @@ -11,7 +11,8 @@ General tips: [below](#secondary-dependencies) for more details. ## Identifiers -Each package is required to define at least these variables: +If package does not define a `$(package)_local_dir` variable, it is required to +define these variables: $(package)_version: Version of the upstream library or program. If there is no version, a @@ -28,6 +29,9 @@ Each package is required to define at least these variables: $(package)_sha256_hash: The sha256 hash of the upstream file +If a package does define a `$(package)_local_dir` variable, the above variables +are not required and will be ignored. + These variables are optional: $(package)_build_subdir: @@ -48,6 +52,18 @@ These variables are optional: Any extra files that will be fetched via $(package)_fetch_cmds. These are specified so that they can be fetched and verified via 'make download'. +## Local packages + +If a package defines a `$(package)_local_dir` variable, the specified directory +will be treated as a download source, and a tarball of its contents will be +saved to `sources/`. A hash of the tarball will also become part of the package +build id, so if the directory contents change, the package and everything +depending on it will be rebuilt. For efficiency, the tarball is cached once it +has been created, but if the local directory is touched, it will be rebuilt. + +Local packages can be useful for using git submodules or subtrees to manage +package sources, or for testing local changes that are not available to +download from an external source. ## Build Variables: After defining the main identifiers, build variables may be added or customized diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk index 47064a9bb6c..158f57b3c68 100644 --- a/depends/packages/libmultiprocess.mk +++ b/depends/packages/libmultiprocess.mk @@ -1,5 +1,6 @@ package=libmultiprocess $(package)_version=$(native_$(package)_version) +$(package)_local_dir=$(native_$(package)_local_dir) $(package)_download_path=$(native_$(package)_download_path) $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index d0db7ba6a2c..f494f6e8a58 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -1,8 +1,5 @@ package=native_libmultiprocess -$(package)_version=35944ffd23fa26652b82210351d50e896ce16c8f -$(package)_download_path=https://github.com/bitcoin-core/libmultiprocess/archive -$(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=b542f270c076d0287c124e7f97b21ab7e32b979ff182491c157c2da9dec723c4 +$(package)_local_dir=../src/ipc/libmultiprocess $(package)_dependencies=native_capnp define $(package)_config_cmds