Allow http workers to send data optimistically as an optimization

src/httpserver.cpp

@@ -1005,9 +1005,11 @@ void HTTPRequest::WriteReply(HTTPStatusCode status, std::span<const std::byte> r
     const std::string headers{res.StringifyHeaders()};
     const auto headers_bytes{std::as_bytes(std::span(headers.begin(), headers.end()))};
 
+    bool send_buffer_was_empty{false};
     // Fill the send buffer with the complete serialized response headers + body
     {
         LOCK(m_client->m_send_mutex);
+        send_buffer_was_empty = m_client->m_send_buffer.empty();
         m_client->m_send_buffer.insert(m_client->m_send_buffer.end(), headers_bytes.begin(), headers_bytes.end());
 
         // We've been using std::span up until now but it is finally time to copy
@@ -1016,10 +1018,6 @@ void HTTPRequest::WriteReply(HTTPStatusCode status, std::span<const std::byte> r
         m_client->m_send_buffer.insert(m_client->m_send_buffer.end(), reply_body.begin(), reply_body.end());
     }
 
-    // Inform Sockman I/O there is data that is ready to be sent to this client
-    // in the next loop iteration.
-    m_client->m_send_ready = true;
-
     LogDebug(
         BCLog::HTTP,
         "HTTPResponse (status code: %d size: %lld) added to send buffer for client %s (id=%lld)\n",
@@ -1027,6 +1025,18 @@ void HTTPRequest::WriteReply(HTTPStatusCode status, std::span<const std::byte> r
         headers_bytes.size() + reply_body.size(),
         m_client->m_origin,
         m_client->m_node_id);
+
+    // If the send buffer was empty before we wrote this reply, we can try an
+    // optimistic send akin to CConnman::PushMessage() in which we
+    // push the data directly out the socket to client right now, instead
+    // of waiting for the next iteration of the Sockman I/O loop.
+    if (send_buffer_was_empty) {
+        m_client->SendBytesFromBuffer();
+    } else {
+        // Inform Sockman I/O there is data that is ready to be sent to this client
+        // in the next loop iteration.
+        m_client->m_send_ready = true;
+    }
 }
 
 bool HTTPClient::ReadRequest(std::unique_ptr<HTTPRequest>& req)
@@ -1092,6 +1102,9 @@ bool HTTPClient::SendBytesFromBuffer()
                 m_disconnect = true;
                 return false;
             }
+        } else {
+            m_send_ready = true;
+            m_prevent_disconnect = true;
         }
     }
 

src/httpserver.h

@@ -307,15 +307,17 @@ public:
     std::atomic_bool m_send_ready{false};
 
     // Set to true when we receive request data and set to false once m_send_buffer is cleared.
-    // Checked during DisconnectClients(). All of these operations take place in the Sockman I/O loop.
+    // Checked during DisconnectClients(). All of these operations take place in the Sockman I/O loop,
-    bool m_prevent_disconnect{false};
+    // however it may get set my a worker thread during an "optimistic send".
+    std::atomic_bool m_prevent_disconnect{false};
 
     // Client request to keep connection open after all requests have been responded to.
     // Set by (potentially multiple) worker threads and checked in the Sockman I/O loop.
     std::atomic_bool m_keep_alive{false};
 
-    // Flag this client for disconnection on next loop
+    // Flag this client for disconnection on next loop.
-    bool m_disconnect{false};
+    // Checked at the end of every Sockman I/O loop, may be set a worker thread.
+    std::atomic_bool m_disconnect{false};
 
     explicit HTTPClient(NodeId node_id, CService addr) : m_node_id(node_id), m_addr(addr)
     {