Merge 44fbc9d83d into c5e44a0435

32d55e28af test: Use the correct node for doubled keypath test (Ava Chow)

Pull request description:

  #29124 had a silent merge conflict with #32350 which resulted in it using the wrong node. Fix the test to use the correct v22 node.

ACKs for top commit:
  maflcko:
    lgtm ACK 32d55e28af
  rkrux:
    ACK 32d55e28af
  BrandonOdiwuor:
    Code Review ACK 32d55e28af

Tree-SHA512: 1e0231985beb382b16e1d608c874750423d0502388db0c8ad450b22d17f9d96f5e16a6b44948ebda5efc750f62b60d0de8dd20131f449427426a36caf374af92

contrib/codegen/generate_client.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import subprocess
+from jinja2 import Environment, FileSystemLoader
+
+def get_schema():
+    """Get the schema from the Bitcoin Core RPC help."""
+    try:
+        result = subprocess.run(['bitcoin-cli', 'schema'], capture_output=True, text=True, check=True)
+        schema = json.loads(result.stdout)
+        return schema
+    except Exception as e:
+        raise RuntimeError("Failed to get schema from bitcoin-cli: " + str(e))
+    
+def sanitize_argument_names(rpcs):
+    for rpc_name, details in rpcs.items():
+        if "argument_names" in details:
+            sanitized = []
+            for arg in details["argument_names"]:
+                sanitized.append(arg.split("|")[0])
+            details["argument_names"] = sanitized
+    
+    return rpcs
+    
+def generate_client(schema, language, output_dir):
+    """Generate client code using the specified language template."""
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    templates_path = os.path.join(script_dir, "templates")
+    env = Environment(loader=FileSystemLoader(templates_path))
+
+    # choose template based on target language
+    if language.lower() == "python":
+        template_file = env.get_template("python_client.jinja2")
+        output_file_name = "bitcoin_rpc_client.py"
+    elif language.lower() == "cpp":
+        template_file = env.get_template("cpp_client.jinja2")
+        output_file_name = "bitcoin_rpc_client.cpp"
+    else:
+        raise ValueError("Unsupported language: " + language)
+    
+    template = env.get_template(template_file)
+    rendered_code = template.render(rpcs=schema.get("rpcs", {}))
+
+    os.makedirs(output_dir, exist_ok=True)
+    output_path = os.path.join(output_dir, output_file_name)
+    with open(output_path, "w") as f:
+        f.write(rendered_code)
+    print(f"Generated {language.upper()} client code at {output_path}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate Bitcoin Core RPC client code.")
+    parser.add_argument('--language', required=True, choices=["python", "cpp"], help="The target language for the client code.")
+    parser.add_argument('--output-dir', default="generated", help="The directory to write the generated code to.")
+    args = parser.parse_args()
+
+    print("Getting schema from Bitcoin Core RPC...")
+    schema = get_schema()
+    rpcs = schema.get("rpcs", {})
+    rpcs = sanitize_argument_names(rpcs)
+    generate_client(schema, args.language, args.output_dir)
+
+if __name__ == "__main__":
+    main()

contrib/codegen/templates/cpp_client.jinja2

@@ -0,0 +1,84 @@
+/**
+ * Auto-generated Bitcoin RPC client (C++)
+ */
+#ifndef BITCOIN_RPC_CLIENT_H
+#define BITCOIN_RPC_CLIENT_H
+
+#include <string>
+#include <stdexcept>
+#include <sstream>
+#include <vector>
+#include <curl/curl.h>
+#include <json/json.h>
+#include <iostream>
+
+class BitcoinRPCClient {
+public:
+    BitcoinRPCClient(const std::string &rpc_user, const std::string &rpc_password,
+                     const std::string &host = "127.0.0.1", int port = 8332)
+        : m_rpcUser(rpc_user), m_rpcPassword(rpc_password), m_host(host), m_port(port) {}
+
+    /**
+     * Send a JSON-RPC call to the Bitcoin node.
+     */
+    std::string call(const std::string &method, const std::string &params_json) {
+        CURL *curl = curl_easy_init();
+        if (!curl) {
+            throw std::runtime_error("Failed to initialize CURL");
+        }
+        std::stringstream url;
+        url << "http://" << m_host << ":" << m_port;
+
+        // Build JSON-RPC payload
+        Json::Value payload;
+        payload["method"] = method;
+        Json::Reader reader;
+        Json::Value params;
+        if (!reader.parse(params_json, params)) {
+            params = Json::Value(Json::arrayValue);
+        }
+        payload["params"] = params;
+        payload["id"] = 1;
+        Json::StreamWriterBuilder writer;
+        std::string payload_str = Json::writeString(writer, payload);
+
+        curl_easy_setopt(curl, CURLOPT_URL, url.str().c_str());
+        curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload_str.c_str());
+        std::string auth = m_rpcUser + ":" + m_rpcPassword;
+        curl_easy_setopt(curl, CURLOPT_USERPWD, auth.c_str());
+        // NOTE: For demonstration purposes, we do not capture the response body.
+        curl_easy_perform(curl);
+        curl_easy_cleanup(curl);
+        return "{}"; // Dummy response for demonstration.
+    }
+
+    {%- for rpc_name, details in rpcs.items() %}
+    /**
+     * {{ details.description | replace("\n", "\n * ") }}
+     {%- if details.argument_names|length > 0 %}
+     * Arguments:
+     {%- for arg in details.argument_names %}
+     *   - {{ arg }}
+     {%- endfor %}
+     {%- endif %}
+     */
+    std::string {{ rpc_name }}({% if details.argument_names|length > 0 %}{% for arg in details.argument_names %}const std::string &{{ arg }}{% if not loop.last %}, {% endif %}{% endfor %}{% endif %}) {
+        // Build JSON array of parameters
+        Json::Value params(Json::arrayValue);
+        {%- for arg in details.argument_names %}
+        params.append({{ arg }});
+        {%- endfor %}
+        Json::StreamWriterBuilder writer;
+        std::string params_json = Json::writeString(writer, params);
+        return call("{{ rpc_name }}", params_json);
+    }
+    {%- endfor %}
+
+private:
+    std::string m_rpcUser;
+    std::string m_rpcPassword;
+    std::string m_host;
+    int m_port;
+};
+
+#endif // BITCOIN_RPC_CLIENT_H

contrib/codegen/templates/python_client.jinja2

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""
+Auto-generated Bitcoin RPC client (Python)
+"""
+
+import requests
+import json
+
+class BitcoinRPCClient:
+    def __init__(self, rpc_user, rpc_password, host="127.0.0.1", port=8332):
+        self.url = f"http://{host}:{port}"
+        self.auth = (rpc_user, rpc_password)
+
+    def call(self, method, params):
+        payload = {"method": method, "params": params, "id": 1}
+        headers = {"Content-Type": "application/json"}
+        response = requests.post(self.url, auth=self.auth, headers=headers, data=json.dumps(payload))
+        return response.json()
+
+    {% for rpc_name, details in rpcs.items() %}
+    def {{ rpc_name }}(self{% if details.argument_names|length > 0 %}, {% for arg in details.argument_names %}{{ arg }}{% if not loop.last %}, {% endif %}{% endfor %}{% endif %}):
+        """
+        {{ details.description }}
+        {% if details.argument_names|length > 0 %}
+        Arguments:
+        {% for arg in details.argument_names %}
+            - {{ arg }}
+        {% endfor %}
+        {% endif %}
+        """
+        params = [{% for arg in details.argument_names %}{{ arg }}{% if not loop.last %}, {% endif %}{% endfor %}]
+        return self.call("{{ rpc_name }}", params)
+
+    {% endfor %}
+
+if __name__ == '__main__':
+    # Example usage:
+    client = BitcoinRPCClient("rpcuser", "rpcpassword")
+    result = client.getblockchaininfo()
+    print(result)

contrib/codegen/test.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+import os
+import tempfile
+import json
+import unittest
+from subprocess import run, PIPE
+from generate_client import load_schema, generate_client, sanitize_argument_names
+
+class TestGenerateClient(unittest.TestCase):
+
+    def setUp(self):
+        # Create a temporary directory for output files
+        self.temp_dir = tempfile.TemporaryDirectory()
+        # Create a sample schema dictionary
+        self.sample_schema = {
+            "rpcs": {
+                "getblockchaininfo": {
+                    "description": "Returns blockchain info.",
+                    "argument_names": []
+                },
+                "addconnection": {
+                    "description": "Opens an outbound connection.",
+                    "argument_names": ["address", "connection_type", "v2transport"]
+                }
+            }
+        }
+        # Sanitize the sample schema if necessary
+        self.sample_schema["rpcs"] = sanitize_argument_names(self.sample_schema["rpcs"])
+
+    def tearDown(self):
+        # Clean up temporary directory
+        self.temp_dir.cleanup()
+
+    def test_generate_cpp_client(self):
+        # Test generating C++ code
+        output_dir = self.temp_dir.name
+        generate_client(self.sample_schema, "cpp", output_dir)
+        generated_file = os.path.join(output_dir, "bitcoin_rpc_client.cpp")
+        self.assertTrue(os.path.exists(generated_file))
+        with open(generated_file, "r") as f:
+            content = f.read()
+        # Check that key parts appear in the generated file
+        self.assertIn("class BitcoinRPCClient", content)
+        self.assertIn("std::string getblockchaininfo()", content)
+        self.assertIn("std::string addconnection(", content)
+
+    def test_generate_python_client(self):
+        # Test generating Python code
+        output_dir = self.temp_dir.name
+        generate_client(self.sample_schema, "python", output_dir)
+        generated_file = os.path.join(output_dir, "bitcoin_rpc_client.py")
+        self.assertTrue(os.path.exists(generated_file))
+        with open(generated_file, "r") as f:
+            content = f.read()
+        # Check that key parts appear in the generated file
+        self.assertIn("class BitcoinRPCClient", content)
+        self.assertIn("def getblockchaininfo(self):", content)
+        self.assertIn("def addconnection(self, address, connection_type, v2transport):", content)
+
+if __name__ == '__main__':
+    unittest.main()

src/CMakeLists.txt

@@ -297,6 +297,7 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL
   rpc/node.cpp
   rpc/output_script.cpp
   rpc/rawtransaction.cpp
+  rpc/schema.cpp
   rpc/server.cpp
   rpc/server_util.cpp
   rpc/signmessage.cpp

src/rpc/schema.cpp

@@ -0,0 +1,220 @@
+#include <rpc/schema.h>
+
+#include <rpc/server.h>
+#include <rpc/util.h>
+#include <univalue.h>
+#include <util/string.h>
+
+using util::SplitString;
+
+// Notes
+// =====
+//
+// This file implements the `schema` RPC. See `Schema::Commands` for the entry
+// point. This RPC is intended to facilite writing code generators which can
+// generate Bitcoin Core RPC clients in other languages. It is as
+// self-contained as possible in this file, to facilitate back-porting to older
+// versions and rebasing onto newer versions.
+//
+// We should probably use something like Open RPC, but the Bitcoin Core RPC API
+// is weird enough that this may be difficult.
+//
+// The returned JSON includes all avaialable information about the RPC, whether
+// useful to external callers or not. There is certainly more detail than
+// necessary, and some of it should probably be elided.
+//
+// The top level type is a map of strings to `vector<CRPCCommand>`. This is
+// because commands can have aliases, at least according to the types. However,
+// I haven't actually seen one, so we just assert that there are no aliases so
+// we don't have to worry about it.
+
+class Schema {
+public:
+    static UniValue Commands(const std::map<std::string, std::vector<const CRPCCommand*>>& commands) {
+        UniValue value{UniValue::VOBJ};
+
+        UniValue rpcs{UniValue::VOBJ};
+
+        for (const auto& entry: commands) {
+            assert(entry.second.size() == 1);
+
+            const auto& command = entry.second[0];
+
+            RPCHelpMan rpc = ((RpcMethodFnType)command->unique_id)();
+
+            rpcs.pushKV(entry.first, Schema::Command(command->category, rpc, command->argNames));
+        }
+
+        value.pushKV("rpcs", rpcs);
+
+        return value;
+    }
+
+private:
+    static UniValue Argument(const RPCArg& argument) {
+        UniValue value{UniValue::VOBJ};
+
+        UniValue names{UniValue::VARR};
+        for (auto const& name: SplitString(argument.m_names, '|')) {
+            names.push_back(name);
+        }
+        value.pushKV("names", names);
+
+        value.pushKV("description", argument.m_description);
+
+        value.pushKV("oneline_description", argument.m_opts.oneline_description);
+
+        value.pushKV("also_positional", argument.m_opts.also_positional);
+
+        UniValue type_str{UniValue::VARR};
+        for (auto const& str: argument.m_opts.type_str) {
+            type_str.push_back(str);
+        }
+        value.pushKV("type_str", type_str);
+
+        bool required = std::holds_alternative<RPCArg::Optional>(argument.m_fallback)
+            && std::get<RPCArg::Optional>(argument.m_fallback) == RPCArg::Optional::NO;
+
+        value.pushKV("required", required);
+
+        if (std::holds_alternative<UniValue>(argument.m_fallback)) {
+            value.pushKV("default", std::get<UniValue>(argument.m_fallback));
+        }
+
+        if (std::holds_alternative<std::string>(argument.m_fallback)) {
+            value.pushKV("default_hint", std::get<std::string>(argument.m_fallback));
+        }
+
+        value.pushKV("hidden", argument.m_opts.hidden);
+
+        value.pushKV("type", Schema::ArgumentType(argument.m_type));
+
+        UniValue inner{UniValue::VARR};
+        for (auto const& argument: argument.m_inner) {
+            inner.push_back(Schema::Argument(argument));
+        }
+        if (!inner.empty()) {
+            value.pushKV("inner", inner);
+        }
+
+        return value;
+    }
+
+    static std::string ArgumentType(const RPCArg::Type& type) {
+        switch (type) {
+            case RPCArg::Type::AMOUNT:
+                return "amount";
+            case RPCArg::Type::ARR:
+                return "array";
+            case RPCArg::Type::BOOL:
+                return "boolean";
+            case RPCArg::Type::NUM:
+                return "number";
+            case RPCArg::Type::OBJ:
+                return "object";
+            case RPCArg::Type::OBJ_NAMED_PARAMS:
+                return "object";
+            case RPCArg::Type::OBJ_USER_KEYS:
+                return "object";
+            case RPCArg::Type::RANGE:
+                return "range";
+            case RPCArg::Type::STR:
+                return "string";
+            case RPCArg::Type::STR_HEX:
+                return "hex";
+            default:
+                NONFATAL_UNREACHABLE();
+        }
+    }
+
+    static UniValue Command(
+        const std::string& category,
+        const RPCHelpMan& command,
+        const std::vector<std::pair<std::string, bool>>& argNames
+    ) {
+        UniValue value{UniValue::VOBJ};
+
+        value.pushKV("category", category);
+        value.pushKV("description", command.m_description);
+        value.pushKV("examples", command.m_examples.m_examples);
+        value.pushKV("name", command.m_name);
+
+        UniValue argument_names{UniValue::VARR};
+        for (const auto& pair : argNames) {
+            UniValue argument_name{UniValue::VARR};
+            argument_names.push_back(pair.first);
+        }
+        value.pushKV("argument_names", argument_names);
+
+        UniValue arguments{UniValue::VARR};
+        for (const auto& argument : command.m_args) {
+            arguments.push_back(Schema::Argument(argument));
+        }
+        value.pushKV("arguments", arguments);
+
+        UniValue results{UniValue::VARR};
+        for (const auto& result : command.m_results.m_results) {
+            results.push_back(Schema::Result(result));
+        }
+        value.pushKV("results", results);
+
+        return value;
+    }
+
+    static UniValue Result(const RPCResult& result) {
+        UniValue value{UniValue::VOBJ};
+        value.pushKV("type", Schema::ResultType(result.m_type));
+        value.pushKV("optional", result.m_optional);
+        value.pushKV("description", result.m_description);
+        value.pushKV("skip_type_check", result.m_skip_type_check);
+        value.pushKV("key_name", result.m_key_name);
+        value.pushKV("condition", result.m_cond);
+
+        UniValue inner{UniValue::VARR};
+        for (auto const& result: result.m_inner) {
+            inner.push_back(Schema::Result(result));
+        }
+        if (!inner.empty()) {
+            value.pushKV("inner", inner);
+        }
+
+        return value;
+    }
+
+    static std::string ResultType(const RPCResult::Type& type) {
+        switch (type) {
+            case RPCResult::Type::OBJ:
+                return "object";
+            case RPCResult::Type::ARR:
+                return "array";
+            case RPCResult::Type::STR:
+                return "string";
+            case RPCResult::Type::NUM:
+                return "number";
+            case RPCResult::Type::BOOL:
+                return "boolean";
+            case RPCResult::Type::NONE:
+                return "none";
+            case RPCResult::Type::ANY:
+                return "any";
+            case RPCResult::Type::STR_AMOUNT:
+                return "amount";
+            case RPCResult::Type::STR_HEX:
+                return "hex";
+            case RPCResult::Type::OBJ_DYN:
+                return "object";
+            case RPCResult::Type::ARR_FIXED:
+                return "object";
+            case RPCResult::Type::NUM_TIME:
+                return "timestamp";
+            case RPCResult::Type::ELISION:
+                return "elision";
+            default:
+                NONFATAL_UNREACHABLE();
+        }
+    }
+};
+
+UniValue CommandSchemas(const std::map<std::string, std::vector<const CRPCCommand*>>& commands) {
+    return Schema::Commands(commands);
+}

src/rpc/schema.h

@@ -0,0 +1,15 @@
+#ifndef BITCOIN_RPC_SCHEMA_H
+#define BITCOIN_RPC_SCHEMA_H
+
+#include <map>
+#include <string>
+#include <vector>
+
+class CRPCCommand;
+class UniValue;
+
+class Schema;
+
+UniValue CommandSchemas(const std::map<std::string, std::vector<const CRPCCommand*>>& commands);
+
+#endif // BITCOIN_RPC_SCHEMA_H

src/rpc/server.cpp

@@ -12,6 +12,7 @@
 #include <logging.h>
 #include <node/context.h>
 #include <node/kernel_notifications.h>
+#include <rpc/schema.h>
 #include <rpc/server_util.h>
 #include <rpc/util.h>
 #include <sync.h>
@@ -124,6 +125,12 @@ std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest&
     return strRet;
 }
 
+UniValue CRPCTable::schema() const
+{
+    return CommandSchemas(this->mapCommands);
+}
+
+
 static RPCHelpMan help()
 {
     return RPCHelpMan{"help",
@@ -152,6 +159,22 @@ static RPCHelpMan help()
     };
 }
 
+static RPCHelpMan schema()
+{
+    return RPCHelpMan{"schema",
+                "\nReturn RPC command JSON Schema descriptions.\n",
+                {},
+                {
+                    RPCResult{RPCResult::Type::OBJ, "", "FOO"},
+                },
+                RPCExamples{""},
+        [&](const RPCHelpMan& self, const JSONRPCRequest& jsonRequest) -> UniValue
+{
+    return tableRPC.schema();
+},
+    };
+}
+
 static RPCHelpMan stop()
 {
     static const std::string RESULT{CLIENT_NAME " stopping"};
@@ -246,6 +269,7 @@ static const CRPCCommand vRPCCommands[]{
     /* Overall control/query calls */
     {"control", &getrpcinfo},
     {"control", &help},
+    {"control", &schema},
     {"control", &stop},
     {"control", &uptime},
 };

src/rpc/server.h

@@ -164,6 +164,8 @@ public:
      */
     void appendCommand(const std::string& name, const CRPCCommand* pcmd);
     bool removeCommand(const std::string& name, const CRPCCommand* pcmd);
+
+    UniValue schema() const;
 };
 
 bool IsDeprecatedRPCEnabled(const std::string& method);

src/rpc/util.h

@@ -12,6 +12,7 @@
 #include <pubkey.h>
 #include <rpc/protocol.h>
 #include <rpc/request.h>
+#include <rpc/schema.h>
 #include <script/script.h>
 #include <script/sign.h>
 #include <uint256.h>
@@ -503,6 +504,7 @@ private:
     R ArgValue(size_t i) const;
     //! Return positional index of a parameter using its name as key.
     size_t GetParamIndex(std::string_view key) const;
+    friend Schema;
 };
 
 /**

test/functional/wallet_backwards_compatibility.py

@@ -87,7 +87,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework):
         # 0.21.x and 22.x would both produce bad derivation paths when topping up an inactive hd chain
         # Make sure that this is being automatically cleaned up by migration
         node_master = self.nodes[1]
-        node_v22 = self.nodes[self.num_nodes - 5]
+        node_v22 = self.nodes[self.num_nodes - 3]
         wallet_name = "bad_deriv_path"
         node_v22.createwallet(wallet_name=wallet_name, descriptors=False)
         bad_deriv_wallet = node_v22.get_wallet_rpc(wallet_name)