From f248024b6559b56852179c2a3df64b4eb884bcc5 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 30 Dec 2024 18:31:52 -0300 Subject: [PATCH] Merge squash 'unixfox:invidious-companion': MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit a5acddefa92c454fced4a9176df10dc85efdb516 Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon Dec 30 22:25:13 2024 +0100 missing , commit 84b87bedadbd4d35190b1f4d6b3e4fc1abf2440a Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon Dec 30 22:19:45 2024 +0100 fixing format commit bfaf72b3038c3c8cad6d5e68f9f2ad3a49c2a9fc Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon Dec 30 21:52:34 2024 +0100 skip proxy for invidious companion commit f550359ae941d84cdaee0a966ed332354ef18f42 Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon Dec 30 21:52:07 2024 +0100 !empty? to present? commit e9c354d5a34df636306b1819dd17fff9e01b1a1e Author: Émilien (perso) <4016501+unixfox@users.noreply.github.com> Date: Tue Dec 24 17:43:54 2024 +0000 Better doc for invidious_companion_key commit 0dba7675a2c1d51988b3f2911a9fb3a1f91bae52 Author: Émilien (perso) <4016501+unixfox@users.noreply.github.com> Date: Tue Dec 24 16:18:58 2024 +0000 Better document private_url and public_url commit 1de20546182421e1280ec2b68c6d347abead7c54 Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Fri Dec 13 20:08:57 2024 +0100 add ability for invidious companion to check request from invidious commit ab72bbad7afb7d143883a7d0610145f68c06bac8 Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sun Dec 8 22:24:57 2024 +0100 fix ameba Redundant use of `Object#to_s` in interpolation commit a571eeaa381523f5efb29dea0f5fe097f4f1252c Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sun Dec 8 22:22:08 2024 +0100 format watch.cr commit f710dd37bf4327748b43067d75025cc915b5639c Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sun Dec 8 22:21:10 2024 +0100 apply all the suggestions + rework invidious_companion parameter commit 7a070fa710b7807cdda061d413ca9369a0962353 Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon Nov 18 12:30:37 2024 +0100 invidious companion always used so always add CSP and redirect latest_version commit 1f51edd0b915ca64df7f195aa271f74c7ef093cb Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Mon Nov 18 12:22:23 2024 +0100 fix linting commit 734e72503f88f9741279ab385e86f5d2b340c71b Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sun Nov 17 19:18:29 2024 +0100 fix download function when invidious companion used commit bb2e3b2a3e5f53610b9dd602f8507303ec641450 Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sun Nov 17 12:26:35 2024 +0100 crystal handle decompression already by itself commit b51770dbdbdcca04d04849d37e5f11ce20948c73 Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat Nov 16 23:00:48 2024 +0100 fix linting + use .empty? commit 9f846127aea9b4f392acb062d662fff2cc58d1d0 Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat Nov 16 22:38:00 2024 +0100 fixing "end" misplacement commit 1aa154b9787eddcdee960d06aed4c1c91f17c1c3 Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat Nov 16 22:33:28 2024 +0100 separate invidious_companion logic + better config.yaml config commit ff3305d52175c517b035d79b3c0c6a84809cbd0f Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Fri Nov 8 21:05:17 2024 +0100 move config checks for invidious companion commit 409df4cff3cc69c5565a12feb307441eed36f937 Author: Émilien (perso) <4016501+unixfox@users.noreply.github.com> Date: Tue Nov 5 15:50:59 2024 +0100 modify the description for config.example.yaml about invidious companion commit 27b24f51abcccd1c68f4dc1c29c0c62ca26e604c Author: Émilien (perso) <4016501+unixfox@users.noreply.github.com> Date: Tue Nov 5 15:31:45 2024 +0100 Remove debug puts functions Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com> commit 1c9f5b0a2b38ad94fb8972764ffae98df1e41dc9 Author: Émilien (perso) <4016501+unixfox@users.noreply.github.com> Date: Tue Nov 5 15:31:21 2024 +0100 Use sample instead of Random.rand Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com> commit 2cc204a0457665f8e334970d7e54b1843a667ab6 Author: Emilien <4016501+unixfox@users.noreply.github.com> Date: Fri Nov 1 21:30:58 2024 +0100 throw error if inv_sig_helper and invidious_companion used same time commit c612423a4d64f0adbef135074fc55dcc1c362f84 Author: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Mon Oct 21 01:20:16 2024 +0200 fixing condition for Content-Security-Policy commit 195446337159d2cb92b48510af7311fe0cc0f5bb Author: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Sun Oct 20 23:53:08 2024 +0200 fix Shadowing outer local variable `response` commit 73c84baf9fa6eaf9c5d4981bc199f81306ebe5a2 Author: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Sun Oct 20 23:51:00 2024 +0200 redirect latest_version and dash manifest to invidious companion commit 3dff7a76cf9f64ec70aac0a057a3b0bfa1edfc82 Author: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Sun Oct 20 02:10:55 2024 +0200 add support for invidious companion --- config/config.example.yml | 47 +++++++++++++++++++ src/invidious/config.cr | 33 +++++++++++++ src/invidious/helpers/utils.cr | 19 ++++++++ src/invidious/routes/api/manifest.cr | 5 ++ src/invidious/routes/embed.cr | 7 +++ src/invidious/routes/video_playback.cr | 5 ++ src/invidious/routes/watch.cr | 17 +++++-- src/invidious/videos.cr | 6 ++- src/invidious/videos/parser.cr | 44 +++++++++--------- src/invidious/views/components/player.ecr | 12 ++++- src/invidious/yt_backend/connection_pool.cr | 8 ++-- src/invidious/yt_backend/youtube_api.cr | 51 ++++++++++++++++++++- 12 files changed, 222 insertions(+), 32 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index f5beaf81..92571726 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -54,6 +54,53 @@ db: ## #signature_server: +## +## Invidious companion is an external program +## for loading the video streams from YouTube servers. +## +## When this setting is commented out, Invidious companion is not used. +## Otherwise, Invidious will proxy the requests to Invidious companion. +## +## Note: multiple URL can be configured. In this case, invidious will +## randomly pick one every time video data needs to be retrieved. This +## URL is then kept in the video metadata cache to allow video playback +## to work. Once said cache has expired, requesting that video's data +## again will cause a new companion URL to be picked. +## +## The parameter private_url needs to be configured for the internal +## communication between the companion and Invidious. +## And public_url is the public URL from which companion is listening +## to the requests from the user(s). +## +## If you are using a reverse proxy then you will probably need to +## configure the public_url to be the same as the domain used for Invidious. +## Also apply when used from an external IP address (without a domain). +## Examples: https://MYINVIDIOUSDOMAIN or http://192.168.1.100:8282 +## +## Both parameter can have identical URL when Invidious is hosted in +## an internal network or at home or locally (localhost). +## +## Accepted values: "http(s)://:" +## Default: +## +#invidious_companion: +# - private_url: "http://localhost:8282" +# public_url: "http://localhost:8282" + +## +## API key for Invidious companion, used for securing the communication +## between Invidious and Invidious companion. +## The size of the key needs to be more or equal to 16. +## +## Note: This parameter is mandatory when Invidious companion is enabled +## and should be a random string. +## Such random string can be generated on linux with the following +## command: `pwgen 16 1` +## +## Accepted values: a string +## Default: +## +#invidious_companion_key: "CHANGE_ME!!" ######################################### # diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 76b1d46f..35f3503d 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -67,6 +67,16 @@ end class Config include YAML::Serializable + class CompanionConfig + include YAML::Serializable + + @[YAML::Field(converter: Preferences::URIConverter)] + property private_url : URI = URI.parse("") + + @[YAML::Field(converter: Preferences::URIConverter)] + property public_url : URI = URI.parse("") + end + # Number of threads to use for crawling videos from channels (for updating subscriptions) property channel_threads : Int32 = 1 # Time interval between two executions of the job that crawls channel videos (subscriptions update). @@ -175,6 +185,12 @@ class Config # poToken for passing bot attestation property po_token : String? = nil + # Invidious companion + property invidious_companion : Array(CompanionConfig) = [] of CompanionConfig + + # Invidious companion API key + property invidious_companion_key : String = "" + # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new @@ -329,6 +345,23 @@ class Config end {% end %} + if config.invidious_companion.present? + # invidious_companion and signature_server can't work together + if config.signature_server + puts "Config: You can not run inv_sig_helper and invidious_companion at the same time." + exit(1) + elsif config.invidious_companion_key.empty? + puts "Config: Please configure a key if you are using invidious companion." + exit(1) + elsif config.invidious_companion_key == "CHANGE_ME!!" + puts "Config: The value of 'invidious_companion_key' needs to be changed!!" + exit(1) + elsif config.invidious_companion_key.size < 16 + puts "Config: The value of 'invidious_companion_key' needs to be a size of 16 or more." + exit(1) + end + end + # HMAC_key is mandatory # See: https://github.com/iv-org/invidious/issues/3854 if config.hmac_key.empty? diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index a067b142..0c50e751 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -397,3 +397,22 @@ def gen_videoplayback_proxy_list end return external_videoplayback_proxy end + +def encrypt_ecb_without_salt(data, key) + cipher = OpenSSL::Cipher.new("aes-128-ecb") + cipher.encrypt + cipher.key = key + + io = IO::Memory.new + io.write(cipher.update(data)) + io.write(cipher.final) + io.rewind + + return io +end + +def invidious_companion_encrypt(data) + timestamp = Time.utc.to_unix + encrypted_data = encrypt_ecb_without_salt("#{timestamp}|#{data}", CONFIG.invidious_companion_key) + return Base64.urlsafe_encode(encrypted_data) +end diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index c353ea75..729a40b8 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -8,6 +8,11 @@ module Invidious::Routes::API::Manifest id = env.params.url["id"] region = env.params.query["region"]? + if CONFIG.invidious_companion.present? + invidious_companion = CONFIG.invidious_companion.sample + return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}" + end + # Since some implementations create playlists based on resolution regardless of different codecs, # we can opt to only add a source to a representation if it has a unique height within that representation unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 00f24159..49aeade5 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -203,6 +203,13 @@ module Invidious::Routes::Embed return env.redirect url end + if companion_base_url = video.invidious_companion.try &.["baseUrl"].as_s + env.response.headers["Content-Security-Policy"] = + env.response.headers["Content-Security-Policy"] + .gsub("media-src", "media-src #{companion_base_url}") + .gsub("connect-src", "connect-src #{companion_base_url}") + end + rendered "embed" end end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index d301d2e5..5ee5deeb 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -258,6 +258,11 @@ module Invidious::Routes::VideoPlayback # YouTube /videoplayback links expire after 6 hours, # so we have a mechanism here to redirect to the latest version def self.latest_version(env) + if CONFIG.invidious_companion.present? + invidious_companion = CONFIG.invidious_companion.sample + return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}" + end + id = env.params.query["id"]? itag = env.params.query["itag"]?.try &.to_i? diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 10a851ff..c9bb9f7e 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -217,6 +217,13 @@ module Invidious::Routes::Watch video_url = nil end + if companion_base_url = video.invidious_companion.try &.["baseUrl"].as_s + env.response.headers["Content-Security-Policy"] = + env.response.headers["Content-Security-Policy"] + .gsub("media-src", "media-src #{companion_base_url}") + .gsub("connect-src", "connect-src #{companion_base_url}") + end + templated "watch" end @@ -347,14 +354,18 @@ module Invidious::Routes::Watch env.params.query["label"] = URI.decode_www_form(label.as_s) return Invidious::Routes::API::V1::Videos.captions(env) - elsif itag = download_widget["itag"]?.try &.as_i + elsif itag = download_widget["itag"]?.try &.as_i.to_s # URL params specific to /latest_version env.params.query["id"] = video_id - env.params.query["itag"] = itag.to_s env.params.query["title"] = filename env.params.query["local"] = "true" - return Invidious::Routes::VideoPlayback.latest_version(env) + if (CONFIG.invidious_companion.present?) + video = get_video(video_id) + return env.redirect "#{video.invidious_companion["baseUrl"].as_s}/latest_version?#{env.params.query}" + else + return Invidious::Routes::VideoPlayback.latest_version(env) + end else return error_template(400, "Invalid label or itag") end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index c6e69ee5..4a1dcb50 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -15,7 +15,7 @@ struct Video # NOTE: don't forget to bump this number if any change is made to # the `params` structure in videos/parser.cr!!! # - SCHEMA_VERSION = 2 + SCHEMA_VERSION = 3 property id : String @@ -192,6 +192,10 @@ struct Video } end + def invidious_companion : Hash(String, JSON::Any)? + info["invidiousCompanion"]?.try &.as_h || {} of String => JSON::Any + end + # Macros defining getters/setters for various types of data private macro getset_string(name) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index a0888bfa..e645cc66 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -100,30 +100,32 @@ def extract_video_info(video_id : String) params = parse_video_info(video_id, player_response) params["reason"] = JSON::Any.new(reason) if reason - new_player_response = nil + if CONFIG.invidious_companion.present? + new_player_response = nil - # Don't use Android test suite client if po_token is passed because po_token doesn't - # work for Android test suite client. - if reason.nil? && CONFIG.po_token.nil? - # Fetch the video streams using an Android client in order to get the - # decrypted URLs and maybe fix throttling issues (#2194). See the - # following issue for an explanation about decrypted URLs: - # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 - client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite - new_player_response = try_fetch_streaming_data(video_id, client_config) + # Don't use Android test suite client if po_token is passed because po_token doesn't + # work for Android test suite client. + if reason.nil? && CONFIG.po_token.nil? + # Fetch the video streams using an Android client in order to get the + # decrypted URLs and maybe fix throttling issues (#2194). See the + # following issue for an explanation about decrypted URLs: + # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 + client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite + new_player_response = try_fetch_streaming_data(video_id, client_config) + end + + # Replace player response and reset reason + if !new_player_response.nil? + # Preserve captions & storyboard data before replacement + new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]? + new_player_response["captions"] = player_response["captions"] if player_response["captions"]? + + player_response = new_player_response + params.delete("reason") + end end - # Replace player response and reset reason - if !new_player_response.nil? - # Preserve captions & storyboard data before replacement - new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]? - new_player_response["captions"] = player_response["captions"] if player_response["captions"]? - - player_response = new_player_response - params.delete("reason") - end - - {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f| + {"captions", "playabilityStatus", "playerConfig", "storyboards", "invidiousCompanion"}.each do |f| params[f] = player_response[f] if player_response[f]? end diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 5c28358b..d7589381 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -22,6 +22,8 @@ audio_streams.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local + src_url = video.invidious_companion["baseUrl"].as_s + src_url + + "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?) bitrate = fmt["bitrate"] mimetype = HTML.escape(fmt["mimeType"].as_s) @@ -34,8 +36,12 @@ <% end %> <% end %> <% else %> - <% if params.quality == "dash" %> - + <% if params.quality == "dash" + src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1" + src_url = video.invidious_companion["baseUrl"].as_s + src_url + + "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?) + %> + <% end %> <% @@ -44,6 +50,8 @@ fmt_stream.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local + src_url = video.invidious_companion["baseUrl"].as_s + src_url + + "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?) quality = fmt["quality"] mimetype = HTML.escape(fmt["mimeType"].as_s) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index c4a73aa7..060e5e2b 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -61,9 +61,9 @@ def add_yt_headers(request) end end -def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false) +def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false, use_http_proxy : Bool = true) client = HTTP::Client.new(url) - client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy && use_http_proxy # Force the usage of a specific configured IP Family if force_resolve @@ -78,8 +78,8 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, force_you return client end -def make_client(url : URI, region = nil, force_resolve : Bool = false, &) - client = make_client(url, region, force_resolve: force_resolve) +def make_client(url : URI, region = nil, force_resolve : Bool = false, use_http_proxy : Bool = true, &) + client = make_client(url, region, force_resolve: force_resolve, use_http_proxy: use_http_proxy) begin yield client ensure diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index beb3f821..6c1c6584 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -491,7 +491,11 @@ module YoutubeAPI data["params"] = params end - return self._post_json("/youtubei/v1/player", data, client_config) + if CONFIG.invidious_companion.present? + return self._post_invidious_companion("/youtubei/v1/player", data) + else + return self._post_json("/youtubei/v1/player", data, client_config) + end end #################################################################### @@ -657,6 +661,51 @@ module YoutubeAPI return initial_data end + #################################################################### + # _post_invidious_companion(endpoint, data) + # + # Internal function that does the actual request to Invidious companion + # and handles errors. + # + # The requested data is an endpoint (URL without the domain part) + # and the data as a Hash object. + # + def _post_invidious_companion( + endpoint : String, + data : Hash + ) : Hash(String, JSON::Any) + headers = HTTP::Headers{ + "Content-Type" => "application/json; charset=UTF-8", + "Authorization" => "Bearer #{CONFIG.invidious_companion_key}", + } + + # Logging + LOGGER.debug("Invidious companion: Using endpoint: \"#{endpoint}\"") + LOGGER.trace("Invidious companion: POST data: #{data}") + + # Send the POST request + + begin + invidious_companion = CONFIG.invidious_companion.sample + response = make_client(invidious_companion.private_url, use_http_proxy: false, + &.post(endpoint, headers: headers, body: data.to_json)) + body = response.body + if (response.status_code != 200) + raise Exception.new( + "Error while communicating with Invidious companion: \ + status code: #{response.status_code} and body: #{body.dump}" + ) + end + rescue ex + raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) + end + + # Convert result to Hash + initial_data = JSON.parse(body).as_h + + return initial_data + end + #################################################################### # _decompress(body_io, headers) #