From eb8fcc9e88bf71694e7d9e40600ef3fd334bc46e Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 25 Aug 2023 16:08:02 -0700 Subject: [PATCH 01/19] Add support for using HTTP proxies --- config/config.example.yml | 11 +++++++++++ shard.lock | 10 +++++++--- shard.yml | 3 +++ src/invidious.cr | 1 + src/invidious/config.cr | 11 +++++++++++ src/invidious/yt_backend/connection_pool.cr | 18 ++++++++++++++++++ 6 files changed, 51 insertions(+), 3 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 38085a20..9fdc4eda 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -160,6 +160,17 @@ https_only: false ## #force_resolve: +## +## Configuration for using a HTTP proxy +## +## If unset, then no HTTP proxy will be used. +## +http_proxy: + user: + password: + host: + port: + ## ## Use Innertube's transcripts API instead of timedtext for closed captions diff --git a/shard.lock b/shard.lock index efb60a59..85928c47 100644 --- a/shard.lock +++ b/shard.lock @@ -6,11 +6,11 @@ shards: athena-negotiation: git: https://github.com/athena-framework/negotiation.git - version: 0.1.1 + version: 0.1.3 backtracer: git: https://github.com/sija/backtracer.cr.git - version: 1.2.1 + version: 1.2.2 db: git: https://github.com/crystal-lang/crystal-db.git @@ -20,6 +20,10 @@ shards: git: https://github.com/crystal-loot/exception_page.git version: 0.2.2 + http_proxy: + git: https://github.com/mamantoha/http_proxy.git + version: 0.10.1 + kemal: git: https://github.com/kemalcr/kemal.git version: 1.1.2 @@ -42,7 +46,7 @@ shards: spectator: git: https://github.com/icy-arctic-fox/spectator.git - version: 0.10.4 + version: 0.10.6 sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git diff --git a/shard.yml b/shard.yml index be06a7df..ddd510d4 100644 --- a/shard.yml +++ b/shard.yml @@ -28,6 +28,9 @@ dependencies: athena-negotiation: github: athena-framework/negotiation version: ~> 0.1.1 + http_proxy: + github: mamantoha/http_proxy + version: ~> 0.10.1 development_dependencies: spectator: diff --git a/src/invidious.cr b/src/invidious.cr index e0bd0101..d4114386 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -23,6 +23,7 @@ require "kilt" require "./ext/kemal_content_for.cr" require "./ext/kemal_static_file_handler.cr" +require "http_proxy" require "athena-negotiation" require "openssl/hmac" require "option_parser" diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 09c2168b..e12054d0 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -54,6 +54,15 @@ struct ConfigPreferences end end +struct HTTPProxyConfig + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 +end + class Config include YAML::Serializable @@ -126,6 +135,8 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # HTTP Proxy configuration + property http_proxy : HTTPProxyConfig? = nil # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index d3dbcc0e..0e4d8aff 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -26,12 +26,14 @@ struct YoutubeConnectionPool def client(&block) conn = pool.checkout + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy begin response = yield conn rescue ex conn.close conn = HTTP::Client.new(url) + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -46,6 +48,7 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = HTTP::Client.new(url) + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -66,6 +69,8 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) client.read_timeout = 10.seconds client.connect_timeout = 10.seconds + client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + return client end @@ -77,3 +82,16 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) client.close end end + +def make_configured_http_proxy_client + # This method is only called when configuration for an HTTP proxy are set + config_proxy = CONFIG.http_proxy.not_nil! + + return HTTP::Proxy::Client.new( + config_proxy.host, + config_proxy.port, + + username: config_proxy.user, + password: config_proxy.password, + ) +end From 3b471ae964ad0122c81205965c221a352e3a658f Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 4 Oct 2023 14:36:04 -0400 Subject: [PATCH 02/19] Automatically initialize proxy via stdlib override --- .../helpers/crystal_class_overrides.cr | 34 +++++++++++++++++++ src/invidious/yt_backend/connection_pool.cr | 7 ++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index bf56d826..71038703 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -18,6 +18,40 @@ end class HTTP::Client property family : Socket::Family = Socket::Family::UNSPEC + # Override stdlib to automatically initialize proxy if configured + # + # Accurate as of crystal 1.10.1 + + def initialize(@host : String, port = nil, tls : TLSContext = nil) + check_host_only(@host) + + {% if flag?(:without_openssl) %} + if tls + raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time" + end + @tls = nil + {% else %} + @tls = case tls + when true + OpenSSL::SSL::Context::Client.new + when OpenSSL::SSL::Context::Client + tls + when false, nil + nil + end + {% end %} + + @port = (port || (@tls ? 443 : 80)).to_i + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + + def initialize(@io : IO, @host = "", @port = 80) + @reconnect = false + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + private def io io = @io return io if io diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 0e4d8aff..f34f48c5 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -26,13 +26,15 @@ struct YoutubeConnectionPool def client(&block) conn = pool.checkout + # Proxy needs to be reinstated every time we get a client from the pool conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + begin response = yield conn rescue ex conn.close - conn = HTTP::Client.new(url) + conn = HTTP::Client.new(url) conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC @@ -48,7 +50,6 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = HTTP::Client.new(url) - conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -69,8 +70,6 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) client.read_timeout = 10.seconds client.connect_timeout = 10.seconds - client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy - return client end From ccb2a6c58ef9b5b7e1c47938cfa4ec574a8560c7 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 28 Apr 2024 21:34:05 -0700 Subject: [PATCH 03/19] Bump http_proxy to v0.10.3 --- shard.lock | 4 ++-- shard.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shard.lock b/shard.lock index 85928c47..99b4e99c 100644 --- a/shard.lock +++ b/shard.lock @@ -6,7 +6,7 @@ shards: athena-negotiation: git: https://github.com/athena-framework/negotiation.git - version: 0.1.3 + version: 0.1.1 backtracer: git: https://github.com/sija/backtracer.cr.git @@ -22,7 +22,7 @@ shards: http_proxy: git: https://github.com/mamantoha/http_proxy.git - version: 0.10.1 + version: 0.10.3 kemal: git: https://github.com/kemalcr/kemal.git diff --git a/shard.yml b/shard.yml index ddd510d4..70ebad0a 100644 --- a/shard.yml +++ b/shard.yml @@ -30,7 +30,7 @@ dependencies: version: ~> 0.1.1 http_proxy: github: mamantoha/http_proxy - version: ~> 0.10.1 + version: ~> 0.10.3 development_dependencies: spectator: From 6b7e7301009e1a9fc2b536bd8d8de04fb8e22ec0 Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 22 May 2024 13:10:46 -0700 Subject: [PATCH 04/19] Validate override for crystal 1.12.1 --- src/invidious/helpers/crystal_class_overrides.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index 71038703..a7d2a5e6 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -20,7 +20,7 @@ class HTTP::Client # Override stdlib to automatically initialize proxy if configured # - # Accurate as of crystal 1.10.1 + # Accurate as of crystal 1.12.1 def initialize(@host : String, port = nil, tls : TLSContext = nil) check_host_only(@host) From 288e1dccda2256a9014364d693b3eb3d7933b242 Mon Sep 17 00:00:00 2001 From: giacomocerquone Date: Thu, 13 Jun 2024 01:10:35 +0200 Subject: [PATCH 05/19] Fix player menus hiding onHover --- assets/css/player.css | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/css/player.css b/assets/css/player.css index 50c7a748..9cb400ad 100644 --- a/assets/css/player.css +++ b/assets/css/player.css @@ -68,6 +68,7 @@ .video-js.player-style-youtube .vjs-menu-button-popup .vjs-menu { margin-bottom: 2em; + padding-top: 2em } .video-js.player-style-youtube .vjs-progress-control .vjs-progress-holder, .video-js.player-style-youtube .vjs-progress-control {height: 5px; From 480e073fa9be184b6839619c38795af582247c19 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Dec 2023 18:20:17 -0800 Subject: [PATCH 06/19] Use HTTP pools for image requests to YouTube --- src/invidious.cr | 8 ++++++++ src/invidious/routes/images.cr | 12 +++++------- src/invidious/yt_backend/connection_pool.cr | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 3804197e..81db2c6c 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -92,6 +92,14 @@ SOFTWARE = { YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) +# Image request pool + +GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size) + +# Mapping of subdomain => YoutubeConnectionPool +# This is needed as we may need to access arbitrary subdomains of ytimg +YTIMG_POOLS = {} of String => YoutubeConnectionPool + # CLI Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index b6a2e110..1964d597 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -32,7 +32,7 @@ module Invidious::Routes::Images } begin - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| + GGPHT_POOL.client &.get(url) do |resp| return request_proc.call(resp) end rescue ex @@ -80,7 +80,7 @@ module Invidious::Routes::Images } begin - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| + get_ytimg_pool(authority).client &.get(url) do |resp| return request_proc.call(resp) end rescue ex @@ -119,7 +119,7 @@ module Invidious::Routes::Images } begin - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| + get_ytimg_pool("i9").client &.get(url) do |resp| return request_proc.call(resp) end rescue ex @@ -165,8 +165,7 @@ module Invidious::Routes::Images if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - # This can likely be optimized into a (small) pool sometime in the future. - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + if get_ytimg_pool("i9").client &.head(thumbnail_resource_path).status_code == 200 name = thumb[:url] + ".jpg" break end @@ -199,8 +198,7 @@ module Invidious::Routes::Images } begin - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| + get_ytimg_pool("i").client &.get(url) do |resp| return request_proc.call(resp) end rescue ex diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index ca612083..26bf2773 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -77,3 +77,18 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &) client.close end end + +# Fetches a HTTP pool for the specified subdomain of ytimg.com +# +# Creates a new one when the specified pool for the subdomain does not exist +def get_ytimg_pool(subdomain) + if pool = YTIMG_POOLS[subdomain]? + return pool + else + LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"") + pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size) + YTIMG_POOLS[subdomain] = pool + + return pool + end +end From 52bc9aa328e44ff32bb1d7f2e05625e4080459c7 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Dec 2023 18:42:40 -0800 Subject: [PATCH 07/19] Refactor duplicate logic in image routes --- src/invidious/routes/images.cr | 97 ++++++++-------------------------- 1 file changed, 21 insertions(+), 76 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 1964d597..7fdd33b0 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -11,29 +11,9 @@ module Invidious::Routes::Images end end - # We're encapsulating this into a proc in order to easily reuse this - # portion of the code for each request block below. - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - return - end - - proxy_file(response, env) - } - begin GGPHT_POOL.client &.get(url) do |resp| - return request_proc.call(resp) + return self.proxy_image(env, resp) end rescue ex end @@ -61,27 +41,9 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Connection"] = "close" - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin get_ytimg_pool(authority).client &.get(url) do |resp| - return request_proc.call(resp) + return self.proxy_image(env, resp) end rescue ex end @@ -101,26 +63,9 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin get_ytimg_pool("i9").client &.get(url) do |resp| - return request_proc.call(resp) + return self.proxy_image(env, resp) end rescue ex end @@ -180,28 +125,28 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin get_ytimg_pool("i").client &.get(url) do |resp| - return request_proc.call(resp) + return self.proxy_image(env, resp) end rescue ex end end + + private def self.proxy_image(env, response) + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 + return env.response.headers.delete("Transfer-Encoding") + end + + return proxy_file(response, env) + end end From 06e1a508e8dc5417a61a02ad1eb08e94fb24ae99 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Dec 2023 18:52:11 -0800 Subject: [PATCH 08/19] Fix headers not being added in image requests Regression from #2364 --- src/invidious/routes/images.cr | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 7fdd33b0..c4197746 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -12,7 +12,7 @@ module Invidious::Routes::Images end begin - GGPHT_POOL.client &.get(url) do |resp| + GGPHT_POOL.client &.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex @@ -42,7 +42,7 @@ module Invidious::Routes::Images end begin - get_ytimg_pool(authority).client &.get(url) do |resp| + get_ytimg_pool(authority).client &.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex @@ -64,7 +64,7 @@ module Invidious::Routes::Images end begin - get_ytimg_pool("i9").client &.get(url) do |resp| + get_ytimg_pool("i9").client &.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex @@ -110,7 +110,7 @@ module Invidious::Routes::Images if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - if get_ytimg_pool("i9").client &.head(thumbnail_resource_path).status_code == 200 + if get_ytimg_pool("i9").client &.head(thumbnail_resource_path, headers).status_code == 200 name = thumb[:url] + ".jpg" break end @@ -126,7 +126,7 @@ module Invidious::Routes::Images end begin - get_ytimg_pool("i").client &.get(url) do |resp| + get_ytimg_pool("i").client &.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex From 4bc77b81bf994336e324d84ab82a362b330c827d Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 23 Dec 2023 13:47:47 -0800 Subject: [PATCH 09/19] Move YTIMG_POOLS to connection_pool.cr --- src/invidious.cr | 4 --- src/invidious/yt_backend/connection_pool.cr | 32 ++++++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 81db2c6c..e0e72415 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -96,10 +96,6 @@ YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size) -# Mapping of subdomain => YoutubeConnectionPool -# This is needed as we may need to access arbitrary subdomains of ytimg -YTIMG_POOLS = {} of String => YoutubeConnectionPool - # CLI Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 26bf2773..646d0d1a 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,17 +1,6 @@ -def add_yt_headers(request) - request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" - - request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["Accept-Language"] ||= "en-us,en;q=0.5" - - # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" - if !CONFIG.cookies.empty? - request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" - end -end +# Mapping of subdomain => YoutubeConnectionPool +# This is needed as we may need to access arbitrary subdomains of ytimg +private YTIMG_POOLS = {} of String => YoutubeConnectionPool struct YoutubeConnectionPool property! url : URI @@ -54,6 +43,21 @@ struct YoutubeConnectionPool end end +def add_yt_headers(request) + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + + request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["Accept-Language"] ||= "en-us,en;q=0.5" + + # Preserve original cookies and add new YT consent cookie for EU servers + request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" + if !CONFIG.cookies.empty? + request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + end +end + def make_client(url : URI, region = nil, force_resolve : Bool = false) client = HTTP::Client.new(url) From 003c6f81dcf6399d1fa808866d0806b915a713ee Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 8 Jan 2024 14:13:38 -0800 Subject: [PATCH 10/19] Preserve connection close header of get_storyboard --- src/invidious/routes/images.cr | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index c4197746..251258ec 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -41,9 +41,14 @@ module Invidious::Routes::Images end end + # A callable proc to be used inside #proxy_image + callable_proc = ->(env : HTTP::Server::Context) { + env.response.headers["Connection"] = "close" + } + begin get_ytimg_pool(authority).client &.get(url, headers) do |resp| - return self.proxy_image(env, resp) + return self.proxy_image(env, resp, callable_proc: callable_proc) end rescue ex end @@ -133,7 +138,7 @@ module Invidious::Routes::Images end end - private def self.proxy_image(env, response) + private def self.proxy_image(env, response, callable_proc = nil) env.response.status_code = response.status_code response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -143,6 +148,10 @@ module Invidious::Routes::Images env.response.headers["Access-Control-Allow-Origin"] = "*" + if callable_proc + callable_proc.call(env) + end + if response.status_code >= 300 return env.response.headers.delete("Transfer-Encoding") end From 75b68618ab14a9f884ee7215a467bc510e8bd2c2 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 25 Apr 2024 13:28:58 -0700 Subject: [PATCH 11/19] Remove useless proc usage in images.cr --- src/invidious/routes/images.cr | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 251258ec..639697db 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -41,14 +41,10 @@ module Invidious::Routes::Images end end - # A callable proc to be used inside #proxy_image - callable_proc = ->(env : HTTP::Server::Context) { - env.response.headers["Connection"] = "close" - } - begin get_ytimg_pool(authority).client &.get(url, headers) do |resp| - return self.proxy_image(env, resp, callable_proc: callable_proc) + env.response.headers["Connection"] = "close" + return self.proxy_image(env, resp) end rescue ex end @@ -138,7 +134,7 @@ module Invidious::Routes::Images end end - private def self.proxy_image(env, response, callable_proc = nil) + private def self.proxy_image(env, response) env.response.status_code = response.status_code response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -148,10 +144,6 @@ module Invidious::Routes::Images env.response.headers["Access-Control-Allow-Origin"] = "*" - if callable_proc - callable_proc.call(env) - end - if response.status_code >= 300 return env.response.headers.delete("Transfer-Encoding") end From 84e4746265d6077b1537b626c9742498f9cb253c Mon Sep 17 00:00:00 2001 From: Fijxu Date: Wed, 18 Sep 2024 18:14:28 -0300 Subject: [PATCH 12/19] SigHelper: Reconnect to signature helper Signed-off-by: Fijxu --- src/invidious/helpers/sig_helper.cr | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr index 9e72c1c7..6d198a42 100644 --- a/src/invidious/helpers/sig_helper.cr +++ b/src/invidious/helpers/sig_helper.cr @@ -175,8 +175,9 @@ module Invidious::SigHelper @queue = {} of TransactionID => Transaction @conn : Connection + @uri_or_path : String - def initialize(uri_or_path) + def initialize(@uri_or_path) @conn = Connection.new(uri_or_path) listen end @@ -186,10 +187,26 @@ module Invidious::SigHelper LOGGER.debug("SigHelper: Multiplexor listening") - # TODO: reopen socket if unexpectedly closed spawn do loop do - receive_data + begin + receive_data + rescue ex + LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...") + # We close the socket because for some reason is not closed. + @conn.close + loop do + begin + @conn = Connection.new(@uri_or_path) + LOGGER.info("SigHelper: Reconnected to SigHelper!") + rescue ex + LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying") + sleep 500.milliseconds + next + end + break if !@conn.closed? + end + end Fiber.yield end end From f51a3b8d2b52f83057d0b3be5686149984e66ada Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 9 Oct 2024 18:07:04 +0200 Subject: [PATCH 13/19] Makefile: Add MT option to enable the 'preview_mt' flag --- Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Makefile b/Makefile index 9eb195df..ec22a0de 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,11 @@ STATIC := 0 NO_DBG_SYMBOLS := 0 +# Enable multi-threading. +# Warning: Experimental feature!! +# invidious is not stable when MT is enabled. +MT := 0 + FLAGS ?= @@ -19,6 +24,10 @@ ifeq ($(STATIC), 1) FLAGS += --static endif +ifeq ($(MT), 1) + FLAGS += -Dpreview_mt +endif + ifeq ($(NO_DBG_SYMBOLS), 1) FLAGS += --no-debug From 952b3625a0a8fb21ab04bc267f94a21c331109f6 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 10 Oct 2024 20:31:22 +0200 Subject: [PATCH 14/19] Add "Filipino (auto-generated)" to the list of caption languages --- locales/en-US.json | 1 + src/invidious/videos/caption.cr | 1 + 2 files changed, 2 insertions(+) diff --git a/locales/en-US.json b/locales/en-US.json index 7827d9c6..c23f6bc3 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -286,6 +286,7 @@ "Esperanto": "Esperanto", "Estonian": "Estonian", "Filipino": "Filipino", + "Filipino (auto-generated)": "Filipino (auto-generated)", "Finnish": "Finnish", "French": "French", "French (auto-generated)": "French (auto-generated)", diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr index 484e61d2..c811cfe1 100644 --- a/src/invidious/videos/caption.cr +++ b/src/invidious/videos/caption.cr @@ -123,6 +123,7 @@ module Invidious::Videos "Esperanto", "Estonian", "Filipino", + "Filipino (auto-generated)", "Finnish", "French", "French (auto-generated)", From 90544e07b6b2a87f464c231215ec34924fadf586 Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Sun, 13 Oct 2024 21:18:21 +0200 Subject: [PATCH 15/19] update the mocks with the latest updated data --- .../videos/regular_videos_extract_spec.cr | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/spec/invidious/videos/regular_videos_extract_spec.cr b/spec/invidious/videos/regular_videos_extract_spec.cr index c647c1d1..f96703f6 100644 --- a/spec/invidious/videos/regular_videos_extract_spec.cr +++ b/spec/invidious/videos/regular_videos_extract_spec.cr @@ -17,8 +17,8 @@ Spectator.describe "parse_video_info" do # Basic video infos expect(info["title"].as_s).to eq("I Gave My 100,000,000th Subscriber An Island") - expect(info["views"].as_i).to eq(126_573_823) - expect(info["likes"].as_i).to eq(5_157_654) + expect(info["views"].as_i).to eq(220_226_287) + expect(info["likes"].as_i).to eq(6_870_691) # For some reason the video length from VideoDetails and the # one from microformat differs by 1s... @@ -48,12 +48,12 @@ Spectator.describe "parse_video_info" do expect(info["relatedVideos"].as_a.size).to eq(20) - expect(info["relatedVideos"][0]["id"]).to eq("Hwybp38GnZw") - expect(info["relatedVideos"][0]["title"]).to eq("I Built Willy Wonka's Chocolate Factory!") + expect(info["relatedVideos"][0]["id"]).to eq("krsBRQbOPQ4") + expect(info["relatedVideos"][0]["title"]).to eq("$1 vs $250,000,000 Private Island!") expect(info["relatedVideos"][0]["author"]).to eq("MrBeast") expect(info["relatedVideos"][0]["ucid"]).to eq("UCX6OQ3DkcsbYNE6H8uQQuVA") - expect(info["relatedVideos"][0]["view_count"]).to eq("179877630") - expect(info["relatedVideos"][0]["short_view_count"]).to eq("179M") + expect(info["relatedVideos"][0]["view_count"]).to eq("230617484") + expect(info["relatedVideos"][0]["short_view_count"]).to eq("230M") expect(info["relatedVideos"][0]["author_verified"]).to eq("true") # Description @@ -76,11 +76,11 @@ Spectator.describe "parse_video_info" do expect(info["ucid"].as_s).to eq("UCX6OQ3DkcsbYNE6H8uQQuVA") expect(info["authorThumbnail"].as_s).to eq( - "https://yt3.ggpht.com/ytc/AL5GRJVuqw82ERvHzsmBxL7avr1dpBtsVIXcEzBPZaloFg=s48-c-k-c0x00ffffff-no-rj" + "https://yt3.ggpht.com/fxGKYucJAVme-Yz4fsdCroCFCrANWqw0ql4GYuvx8Uq4l_euNJHgE-w9MTkLQA805vWCi-kE0g=s48-c-k-c0x00ffffff-no-rj" ) expect(info["authorVerified"].as_bool).to be_true - expect(info["subCountText"].as_s).to eq("143M") + expect(info["subCountText"].as_s).to eq("320M") end it "parses a regular video with no descrition/comments" do @@ -99,8 +99,8 @@ Spectator.describe "parse_video_info" do # Basic video infos expect(info["title"].as_s).to eq("Chris Rea - Auberge") - expect(info["views"].as_i).to eq(10_943_126) - expect(info["likes"].as_i).to eq(0) + expect(info["views"].as_i).to eq(14_324_584) + expect(info["likes"].as_i).to eq(35_870) expect(info["lengthSeconds"].as_i).to eq(283_i64) expect(info["published"].as_s).to eq("2012-05-21T00:00:00Z") @@ -132,14 +132,14 @@ Spectator.describe "parse_video_info" do # Related videos - expect(info["relatedVideos"].as_a.size).to eq(19) + expect(info["relatedVideos"].as_a.size).to eq(20) - expect(info["relatedVideos"][0]["id"]).to eq("Ww3KeZ2_Yv4") - expect(info["relatedVideos"][0]["title"]).to eq("Chris Rea") - expect(info["relatedVideos"][0]["author"]).to eq("PanMusic") - expect(info["relatedVideos"][0]["ucid"]).to eq("UCsKAPSuh1iNbLWUga_igPyA") - expect(info["relatedVideos"][0]["view_count"]).to eq("31581") - expect(info["relatedVideos"][0]["short_view_count"]).to eq("31K") + expect(info["relatedVideos"][0]["id"]).to eq("gUUdQfnshJ4") + expect(info["relatedVideos"][0]["title"]).to eq("Chris Rea - The Road To Hell 1989 Full Version") + expect(info["relatedVideos"][0]["author"]).to eq("NEA ZIXNH") + expect(info["relatedVideos"][0]["ucid"]).to eq("UCYMEOGcvav3gCgImK2J07CQ") + expect(info["relatedVideos"][0]["view_count"]).to eq("53298661") + expect(info["relatedVideos"][0]["short_view_count"]).to eq("53M") expect(info["relatedVideos"][0]["author_verified"]).to eq("false") # Description @@ -156,11 +156,13 @@ Spectator.describe "parse_video_info" do # Author infos - expect(info["author"].as_s).to eq("ChrisReaOfficial") + expect(info["author"].as_s).to eq("ChrisReaVideos") expect(info["ucid"].as_s).to eq("UC_5q6nWPbD30-y6oiWF_oNA") - expect(info["authorThumbnail"].as_s).to be_empty + expect(info["authorThumbnail"].as_s).to eq( + "https://yt3.ggpht.com/ytc/AIdro_n71nsegpKfjeRKwn1JJmK5IVMh_7j5m_h3_1KnUUg=s48-c-k-c0x00ffffff-no-rj" + ) expect(info["authorVerified"].as_bool).to be_false - expect(info["subCountText"].as_s).to eq("-") + expect(info["subCountText"].as_s).to eq("3.11K") end end From e6f52eaf00745732f8c6a6915f7d4df8f84aa29c Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Sun, 13 Oct 2024 23:57:29 +0200 Subject: [PATCH 16/19] update submodule --- mocks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mocks b/mocks index 11ec372f..b55d58de 160000 --- a/mocks +++ b/mocks @@ -1 +1 @@ -Subproject commit 11ec372f72747c09d48ffef04843f72be67d5b54 +Subproject commit b55d58dea94f7144ff0205857dfa70ec14eaa872 From 0d0381870003d8a8286e227043f5d1ddbeacb2fb Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Mon, 14 Oct 2024 00:02:41 +0200 Subject: [PATCH 17/19] libsqlite3-dev is now missing in the CI env --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de538915..1bfa501d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,11 @@ jobs: with: submodules: true + - name: Install required APT packages + run: | + sudo apt install -y libsqlite3-dev + shell: bash + - name: Install Crystal uses: crystal-lang/install-crystal@v1.8.0 with: From d8b893e9ad456598a3e127cec578dd3355141578 Mon Sep 17 00:00:00 2001 From: syeopite <70992037+syeopite@users.noreply.github.com> Date: Fri, 18 Oct 2024 19:33:38 +0000 Subject: [PATCH 18/19] Bump CI matrix (#5015) --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bfa501d..411ec769 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,10 +38,11 @@ jobs: matrix: stable: [true] crystal: - - 1.9.2 - 1.10.1 - 1.11.2 - 1.12.1 + - 1.13.2 + - 1.14.0 include: - crystal: nightly stable: false From 2e3a7ad044b3e37d15d0c87bb33cb85d2d04424f Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 30 Oct 2024 17:13:00 +0100 Subject: [PATCH 19/19] Update CHANGELOG.md --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15991668..f9892e17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ ### Full list of pull requests merged since the last release (newest first) +* Add "Filipino (auto-generated)" to the list of caption languages ([#4995], by @SamantazFox) +* Makefile: Add MT option to enable the 'preview_mt' flag ([#4993], by @SamantazFox) +* SigHelper: Reconnect to signature helper ([#4991], thanks @Fijxu) +* Fix player menus hiding onHover ready ([#4750], thanks @giacomocerquone) +* Use connection pools when requesting images from YouTube ([#4326], thanks @syeopite) +* Add support for using Invidious through a HTTP Proxy ([#4270], thanks @syeopite) * Search: Fix 'youtu.be' URLs in sanitizer ([#4894], by @SamantazFox) * Ameba: Disable Style/RedundantNext rule ([#4888], thanks @syeopite) * Playlists: Fix 'invalid byte sequence' error when subscribing ([#4887], thanks @DmitrySandalov) @@ -22,7 +28,10 @@ [#4122]: https://github.com/iv-org/invidious/pull/4122 [#4193]: https://github.com/iv-org/invidious/pull/4193 +[#4270]: https://github.com/iv-org/invidious/pull/4270 +[#4326]: https://github.com/iv-org/invidious/pull/4326 [#4652]: https://github.com/iv-org/invidious/pull/4652 +[#4750]: https://github.com/iv-org/invidious/pull/4750 [#4850]: https://github.com/iv-org/invidious/pull/4850 [#4862]: https://github.com/iv-org/invidious/pull/4862 [#4863]: https://github.com/iv-org/invidious/pull/4863 @@ -33,6 +42,9 @@ [#4928]: https://github.com/iv-org/invidious/pull/4928 [#4930]: https://github.com/iv-org/invidious/pull/4930 [#4942]: https://github.com/iv-org/invidious/pull/4942 +[#4991]: https://github.com/iv-org/invidious/pull/4991 +[#4993]: https://github.com/iv-org/invidious/pull/4993 +[#4995]: https://github.com/iv-org/invidious/pull/4995 ## v2.20240825.2 (2024-08-26)