Use HTTP pools for image requests to YouTube

src/invidious.cr

@@ -115,6 +115,14 @@ SOFTWARE = {
 
 YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
 
+# Image request pool
+
+GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size)
+
+# Mapping of subdomain => YoutubeConnectionPool
+# This is needed as we may need to access arbitrary subdomains of ytimg
+YTIMG_POOLS = {} of String => YoutubeConnectionPool
+
 # CLI
 Kemal.config.extra_options do |parser|
   parser.banner = "Usage: invidious [arguments]"

src/invidious/routes/images.cr

@@ -32,7 +32,7 @@ module Invidious::Routes::Images
     }
 
     begin
-      HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
+      GGPHT_POOL.client &.get(url) do |resp|
         return request_proc.call(resp)
       end
     rescue ex
@@ -80,7 +80,7 @@ module Invidious::Routes::Images
     }
 
     begin
-      HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
+      get_ytimg_pool(authority).client &.get(url) do |resp|
         return request_proc.call(resp)
       end
     rescue ex
@@ -119,7 +119,7 @@ module Invidious::Routes::Images
     }
 
     begin
-      HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
+      get_ytimg_pool("i9").client &.get(url) do |resp|
         return request_proc.call(resp)
       end
     rescue ex
@@ -165,8 +165,7 @@ module Invidious::Routes::Images
     if name == "maxres.jpg"
       build_thumbnails(id).each do |thumb|
         thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
-        # This can likely be optimized into a (small) pool sometime in the future.
-        if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
+        if get_ytimg_pool("i9").client &.head(thumbnail_resource_path).status_code == 200
           name = thumb[:url] + ".jpg"
           break
         end
@@ -199,8 +198,7 @@ module Invidious::Routes::Images
     }
 
     begin
-      # This can likely be optimized into a (small) pool sometime in the future.
-      HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
+      get_ytimg_pool("i").client &.get(url) do |resp|
         return request_proc.call(resp)
       end
     rescue ex

src/invidious/yt_backend/connection_pool.cr

@@ -82,15 +82,17 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &)
   end
 end
 
-def make_configured_http_proxy_client
-  # This method is only called when configuration for an HTTP proxy are set
-  config_proxy = CONFIG.http_proxy.not_nil!
+# Fetches a HTTP pool for the specified subdomain of ytimg.com
+#
+# Creates a new one when the specified pool for the subdomain does not exist
+def get_ytimg_pool(subdomain)
+  if pool = YTIMG_POOLS[subdomain]?
+    return pool
+  else
+    LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"")
+    pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size)
+    YTIMG_POOLS[subdomain] = pool
 
-  return HTTP::Proxy::Client.new(
-    config_proxy.host,
-    config_proxy.port,
-
-    username: config_proxy.user,
-    password: config_proxy.password,
-  )
+    return pool
+  end
 end