Merge bitcoin/bitcoin#28076: util: Replace std::filesystem with util/fs.h

bbbbdb0cd5 ci: Add filesystem lint check (MarcoFalke)
fada2f9110 refactor: Replace <filesystem> with <util/fs.h> (MarcoFalke)

Pull request description:

  Using `std::filesystem` is problematic:

  * There is a `fs` namespace wrapper for it. So having two ways to achieve the same is confusing.
  * Not using the `fs` wrapper is dangerous and buggy, because it disables known bugs by deleting problematic functions.

  Fix all issues by removing use of it and adding a linter to avoid using it again in the future.

ACKs for top commit:
  TheCharlatan:
    ACK  bbbbdb0cd5
  fanquake:
    ACK bbbbdb0cd5 🦀

Tree-SHA512: 0e2d49742b08eb2635e6fce41485277cb9c40fe20b81017c391d3472a43787db1278a236825714ca1e41c9d2f59913865cfb0c649e3c8ab1fb598c849f80c660

.cirrus.yml

@@ -84,6 +84,9 @@ task:
     memory: 1G
   # For faster CI feedback, immediately schedule the linters
   << : *CREDITS_TEMPLATE
+  test_runner_cache:
+    folder: "/lint_test_runner"
+    fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:test/lint/test_runner)
   python_cache:
     folder: "/python_build"
     fingerprint_script: cat .python-version /etc/os-release

ci/lint/04_install.sh

@@ -33,6 +33,17 @@ export PATH="${PYTHON_PATH}/bin:${PATH}"
 command -v python3
 python3 --version
 
+export LINT_RUNNER_PATH="/lint_test_runner"
+if [ ! -d "${LINT_RUNNER_PATH}" ]; then
+  ${CI_RETRY_EXE} apt-get install -y cargo
+  (
+    cd ./test/lint/test_runner || exit 1
+    cargo build
+    mkdir -p "${LINT_RUNNER_PATH}"
+    mv target/debug/test_runner "${LINT_RUNNER_PATH}"
+  )
+fi
+
 ${CI_RETRY_EXE} pip3 install \
   codespell==2.2.5 \
   flake8==6.1.0 \

ci/lint/06_script.sh

@@ -30,6 +30,7 @@ test/lint/git-subtree-check.sh src/secp256k1
 test/lint/git-subtree-check.sh src/minisketch
 test/lint/git-subtree-check.sh src/leveldb
 test/lint/git-subtree-check.sh src/crc32c
+RUST_BACKTRACE=1 "${LINT_RUNNER_PATH}/test_runner"
 test/lint/check-doc.py
 test/lint/all-lint.py
 

ci/lint/container-entrypoint.sh

@@ -11,6 +11,7 @@ export LC_ALL=C
 git config --global --add safe.directory /bitcoin
 
 export PATH="/python_build/bin:${PATH}"
+export LINT_RUNNER_PATH="/lint_test_runner"
 
 if [ -z "$1" ]; then
   LOCAL_BRANCH=1 bash -ic "./ci/lint/06_script.sh"

src/bitcoin-chainstate.cpp

@@ -26,13 +26,13 @@
 #include <scheduler.h>
 #include <script/sigcache.h>
 #include <util/chaintype.h>
+#include <util/fs.h>
 #include <util/thread.h>
 #include <validation.h>
 #include <validationinterface.h>
 
 #include <cassert>
 #include <cstdint>
-#include <filesystem>
 #include <functional>
 #include <iosfwd>
 #include <memory>
@@ -50,8 +50,8 @@ int main(int argc, char* argv[])
             << "           BREAK IN FUTURE VERSIONS. DO NOT USE ON YOUR ACTUAL DATADIR." << std::endl;
         return 1;
     }
-    std::filesystem::path abs_datadir = std::filesystem::absolute(argv[1]);
-    std::filesystem::create_directories(abs_datadir);
+    fs::path abs_datadir{fs::absolute(argv[1])};
+    fs::create_directories(abs_datadir);
 
 
     // SETUP: Context

src/common/args.cpp

@@ -28,7 +28,6 @@
 #include <cstdint>
 #include <cstdlib>
 #include <cstring>
-#include <filesystem>
 #include <map>
 #include <optional>
 #include <stdexcept>

src/policy/fees.cpp

@@ -1044,8 +1044,8 @@ void CBlockPolicyEstimator::FlushUnconfirmed()
 
 std::chrono::hours CBlockPolicyEstimator::GetFeeEstimatorFileAge()
 {
-    auto file_time = std::filesystem::last_write_time(m_estimation_filepath);
-    auto now = std::filesystem::file_time_type::clock::now();
+    auto file_time{fs::last_write_time(m_estimation_filepath)};
+    auto now{fs::file_time_type::clock::now()};
     return std::chrono::duration_cast<std::chrono::hours>(now - file_time);
 }
 

src/util/fs.h

@@ -184,6 +184,7 @@ static inline path PathFromString(const std::string& string)
  * already exists or is a symlink to an existing directory.
  * This is a temporary workaround for an issue in libstdc++ that has been fixed
  * upstream [PR101510].
+ * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101510
  */
 static inline bool create_directories(const std::filesystem::path& p)
 {

src/util/fs_helpers.cpp

@@ -11,13 +11,11 @@
 
 #include <logging.h>
 #include <sync.h>
-#include <tinyformat.h>
 #include <util/fs.h>
 #include <util/getuniquepath.h>
 #include <util/syserror.h>
 
 #include <cerrno>
-#include <filesystem>
 #include <fstream>
 #include <map>
 #include <memory>
@@ -263,7 +261,7 @@ bool RenameOver(fs::path src, fs::path dest)
 {
 #ifdef __MINGW64__
     // This is a workaround for a bug in libstdc++ which
-    // implements std::filesystem::rename with _wrename function.
+    // implements fs::rename with _wrename function.
     // This bug has been fixed in upstream:
     //  - GCC 10.3: 8dd1c1085587c9f8a21bb5e588dfe1e8cdbba79e
     //  - GCC 11.1: 1dfd95f0a0ca1d9e6cbc00e6cbfd1fa20a98f312

test/lint/README.md

@@ -13,6 +13,14 @@ DOCKER_BUILDKIT=1 docker build -t bitcoin-linter --file "./ci/lint_imagefile" ./
 Building the container can be done every time, because it is fast when the
 result is cached and it prevents issues when the image changes.
 
+test runner
+===========
+
+To run the checks in the test runner outside the docker, use:
+
+```sh
+( cd ./test/lint/test_runner/ && cargo fmt && cargo clippy && cargo run )
+```
 
 check-doc.py
 ============

test/lint/test_runner/Cargo.lock

@@ -0,0 +1,7 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "test_runner"
+version = "0.1.0"

test/lint/test_runner/Cargo.toml

@@ -0,0 +1,12 @@
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit/.
+
+[package]
+name = "test_runner"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

test/lint/test_runner/src/main.rs

@@ -0,0 +1,77 @@
+// Copyright (c) The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or https://opensource.org/license/mit/.
+
+use std::env;
+use std::path::PathBuf;
+use std::process::Command;
+use std::process::ExitCode;
+
+use String as LintError;
+
+/// Return the git command
+fn git() -> Command {
+    Command::new("git")
+}
+
+/// Return stdout
+fn check_output(cmd: &mut std::process::Command) -> Result<String, LintError> {
+    let out = cmd.output().expect("command error");
+    if !out.status.success() {
+        return Err(String::from_utf8_lossy(&out.stderr).to_string());
+    }
+    Ok(String::from_utf8(out.stdout)
+        .map_err(|e| format!("{e}"))?
+        .trim()
+        .to_string())
+}
+
+/// Return the git root as utf8, or panic
+fn get_git_root() -> String {
+    check_output(git().args(["rev-parse", "--show-toplevel"])).unwrap()
+}
+
+fn lint_std_filesystem() -> Result<(), LintError> {
+    let found = git()
+        .args([
+            "grep",
+            "std::filesystem",
+            "--",
+            "./src/",
+            ":(exclude)src/util/fs.h",
+        ])
+        .status()
+        .expect("command error")
+        .success();
+    if found {
+        Err(r#"
+^^^
+Direct use of std::filesystem may be dangerous and buggy. Please include <util/fs.h> and use the
+fs:: namespace, which has unsafe filesystem functions marked as deleted.
+            "#
+        .to_string())
+    } else {
+        Ok(())
+    }
+}
+
+fn main() -> ExitCode {
+    let test_list = [("std::filesystem check", lint_std_filesystem)];
+
+    let git_root = PathBuf::from(get_git_root());
+
+    let mut test_failed = false;
+    for (lint_name, lint_fn) in test_list {
+        // chdir to root before each lint test
+        env::set_current_dir(&git_root).unwrap();
+        if let Err(err) = lint_fn() {
+            println!("{err}\n^---- Failure generated from {lint_name}!");
+            test_failed = true;
+        }
+    }
+    if test_failed {
+        ExitCode::FAILURE
+    } else {
+        ExitCode::SUCCESS
+    }
+}