From 23e92ebe97d95ec00623c42054b6403d7c0aebb2 Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Sun, 20 Oct 2024 02:10:55 +0200 Subject: [PATCH 1/5] add support for invidious companion --- config/config.example.yml | 16 +++++++ src/invidious/config.cr | 3 ++ src/invidious/videos/parser.cr | 64 +++++++++++++------------ src/invidious/yt_backend/youtube_api.cr | 30 ++++++++++-- 4 files changed, 78 insertions(+), 35 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 759b81e0..0d3824a6 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -54,6 +54,22 @@ db: ## #signature_server: +## +## Path to the Invidious companion. +## An external program for loading the video streams from YouTube servers. +## +## When this setting is commented out, Invidious companion is not used. +## +## When this setting is configured and "external_port" is used then +## you need to configure Invidious companion routes into your reverse proxy. +## If "external_port" is not configured then Invidious will proxy the requests +## to Invidious companion. +## +## Accepted values: "http(s)://:" +## Default: +## +#invidious_companion: + ######################################### # diff --git a/src/invidious/config.cr b/src/invidious/config.cr index c1766fbb..0f13f695 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -149,6 +149,9 @@ class Config # poToken for passing bot attestation property po_token : String? = nil + # Invidious companion + property invidious_companion : String? = nil + # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] property cookies : HTTP::Cookies = HTTP::Cookies.new diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index fb8935d9..1b4c1620 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -104,42 +104,44 @@ def extract_video_info(video_id : String) params = parse_video_info(video_id, player_response) params["reason"] = JSON::Any.new(reason) if reason - new_player_response = nil + if CONFIG.invidious_companion.nil? + new_player_response = nil - # Second try in case WEB_CREATOR doesn't work with po_token. - # Only trigger if reason found and po_token configured. - if reason && CONFIG.po_token - client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer - new_player_response = try_fetch_streaming_data(video_id, client_config) - end + # Second try in case WEB_CREATOR doesn't work with po_token. + # Only trigger if reason found and po_token configured. + if reason && CONFIG.po_token + client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer + new_player_response = try_fetch_streaming_data(video_id, client_config) + end - # Don't use Android client if po_token is passed because po_token doesn't - # work for Android client. - if reason.nil? && CONFIG.po_token.nil? - # Fetch the video streams using an Android client in order to get the - # decrypted URLs and maybe fix throttling issues (#2194). See the - # following issue for an explanation about decrypted URLs: - # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 - client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite - new_player_response = try_fetch_streaming_data(video_id, client_config) - end + # Don't use Android client if po_token is passed because po_token doesn't + # work for Android client. + if reason.nil? && CONFIG.po_token.nil? + # Fetch the video streams using an Android client in order to get the + # decrypted URLs and maybe fix throttling issues (#2194). See the + # following issue for an explanation about decrypted URLs: + # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 + client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite + new_player_response = try_fetch_streaming_data(video_id, client_config) + end - # Last hope - # Only trigger if reason found or didn't work wth Android client. - # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token. - if reason && CONFIG.po_token.nil? - client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed - new_player_response = try_fetch_streaming_data(video_id, client_config) - end + # Last hope + # Only trigger if reason found or didn't work wth Android client. + # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token. + if reason && CONFIG.po_token.nil? + client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed + new_player_response = try_fetch_streaming_data(video_id, client_config) + end - # Replace player response and reset reason - if !new_player_response.nil? - # Preserve captions & storyboard data before replacement - new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]? - new_player_response["captions"] = player_response["captions"] if player_response["captions"]? + # Replace player response and reset reason + if !new_player_response.nil? + # Preserve captions & storyboard data before replacement + new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]? + new_player_response["captions"] = player_response["captions"] if player_response["captions"]? - player_response = new_player_response - params.delete("reason") + player_response = new_player_response + params.delete("reason") + end end {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f| diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index baa3cd92..212263da 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -616,12 +616,19 @@ module YoutubeAPI headers = HTTP::Headers{ "Content-Type" => "application/json; charset=UTF-8", - "Accept-Encoding" => "gzip, deflate", "x-goog-api-format-version" => "2", "x-youtube-client-name" => client_config.name_proto, "x-youtube-client-version" => client_config.version, } + if CONFIG.invidious_companion && endpoint == "/youtubei/v1/player" + headers["Authorization"] = "Bearer " + CONFIG.hmac_key + end + + if !CONFIG.invidious_companion + headers["Accept-Encoding"] = "gzip, deflate" + end + if user_agent = client_config.user_agent headers["User-Agent"] = user_agent end @@ -635,11 +642,26 @@ module YoutubeAPI LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") LOGGER.trace("YoutubeAPI: POST data: #{data}") + invidious_companion_url = CONFIG.invidious_companion + # Send the POST request - body = YT_POOL.client() do |client| - client.post(url, headers: headers, body: data.to_json) do |response| - self._decompress(response.body_io, response.headers["Content-Encoding"]?) + if invidious_companion_url && endpoint == "/youtubei/v1/player" + begin + body = make_client(URI.parse(invidious_companion_url), + &.post(endpoint, headers: headers, body: data.to_json).body) + rescue + raise InfoException.new("Unable to communicate with Invidious companion.") end + else + body = YT_POOL.client() do |client| + client.post(url, headers: headers, body: data.to_json) do |response| + self._decompress(response.body_io, response.headers["Content-Encoding"]?) + end + end + end + + if body.nil? && invidious_companion_url + raise InfoException.new("Unable to communicate with Invidious companion.") end # Convert result to Hash From 318fcbf6e4fd90e64885034b66eab09f0fc41acf Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Sun, 20 Oct 2024 23:51:00 +0200 Subject: [PATCH 2/5] redirect latest_version and dash manifest to invidious companion --- config/config.example.yml | 3 ++- src/invidious/config.cr | 2 +- src/invidious/routes/api/manifest.cr | 4 ++++ src/invidious/routes/video_playback.cr | 3 +++ src/invidious/routes/watch.cr | 7 +++++++ src/invidious/videos.cr | 4 ++++ src/invidious/videos/parser.cr | 15 ++++++++++----- src/invidious/views/components/player.ecr | 9 +++++++-- src/invidious/yt_backend/youtube_api.cr | 22 ++++++++++++++-------- 9 files changed, 52 insertions(+), 17 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 0d3824a6..49b97135 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -68,7 +68,8 @@ db: ## Accepted values: "http(s)://:" ## Default: ## -#invidious_companion: +# invidious_companion: +# - http://127.0.0.1:8282 ######################################### diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 0f13f695..3075c93c 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -150,7 +150,7 @@ class Config property po_token : String? = nil # Invidious companion - property invidious_companion : String? = nil + property invidious_companion : Array(String)? = nil # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index d89e752c..43d0eb2f 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -20,6 +20,10 @@ module Invidious::Routes::API::Manifest haltf env, status_code: 403 end + if local && CONFIG.invidious_companion + return env.redirect "#{video.invidious_companion["baseUrl"].as_s}#{env.request.path}?#{env.request.query}" + end + if dashmpd = video.dash_manifest_url response = YT_POOL.client &.get(URI.parse(dashmpd).request_target) diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 24693662..498ede4b 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -294,6 +294,9 @@ module Invidious::Routes::VideoPlayback end if local + if (CONFIG.invidious_companion) + return env.redirect "#{video.invidious_companion["baseUrl"].as_s}#{env.request.path}?#{env.request.query}" + end url = URI.parse(url).request_target.not_nil! url += "&title=#{URI.encode_www_form(title, space_to_plus: false)}" if title end diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index aabe8dfc..961d7b31 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -190,6 +190,13 @@ module Invidious::Routes::Watch captions: video.captions ) + if (CONFIG.invidious_companion && env.params.query["local"] == true) + env.response.headers["Content-Security-Policy"] = + env.response.headers["Content-Security-Policy"] + .gsub("media-src", "media-src " + video.invidious_companion["baseUrl"].as_s) + .gsub("connect-src", "connect-src " + video.invidious_companion["baseUrl"].as_s) + end + templated "watch" end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index ae09e736..131549d0 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -192,6 +192,10 @@ struct Video } end + def invidious_companion : Hash(String, JSON::Any) + info["invidiousCompanion"].try &.as_h + end + # Macros defining getters/setters for various types of data private macro getset_string(name) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 1b4c1620..7a744bdf 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -159,6 +159,10 @@ def extract_video_info(video_id : String) params["streamingData"] = streaming_data end + if CONFIG.invidious_companion + params["invidiousCompanion"] = player_response["invidiousCompanion"] + end + # Data structure version, for cache control params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64) @@ -458,11 +462,12 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any # Music section "music" => JSON.parse(music_list.to_json), # Author infos - "author" => JSON::Any.new(author || ""), - "ucid" => JSON::Any.new(ucid || ""), - "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), - "authorVerified" => JSON::Any.new(author_verified || false), - "subCountText" => JSON::Any.new(subs_text || "-"), + "author" => JSON::Any.new(author || ""), + "ucid" => JSON::Any.new(ucid || ""), + "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""), + "authorVerified" => JSON::Any.new(author_verified || false), + "subCountText" => JSON::Any.new(subs_text || "-"), + "invidiousCompanion" => JSON::Any.new(subs_text), } return params diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 5c28358b..017b3462 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -22,6 +22,7 @@ audio_streams.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local + src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local) bitrate = fmt["bitrate"] mimetype = HTML.escape(fmt["mimeType"].as_s) @@ -34,8 +35,11 @@ <% end %> <% end %> <% else %> - <% if params.quality == "dash" %> - + <% if params.quality == "dash" + src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1" + src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion) + %> + <% end %> <% @@ -44,6 +48,7 @@ fmt_stream.each_with_index do |fmt, i| src_url = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}" src_url += "&local=true" if params.local + src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local) quality = fmt["quality"] mimetype = HTML.escape(fmt["mimeType"].as_s) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 212263da..333d8ab7 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -642,15 +642,21 @@ module YoutubeAPI LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") LOGGER.trace("YoutubeAPI: POST data: #{data}") - invidious_companion_url = CONFIG.invidious_companion + invidious_companion_urls = CONFIG.invidious_companion # Send the POST request - if invidious_companion_url && endpoint == "/youtubei/v1/player" + if invidious_companion_urls && endpoint == "/youtubei/v1/player" + puts "invidious companion section" + puts invidious_companion_urls[Random.rand(invidious_companion_urls.size)] begin - body = make_client(URI.parse(invidious_companion_url), - &.post(endpoint, headers: headers, body: data.to_json).body) - rescue - raise InfoException.new("Unable to communicate with Invidious companion.") + response = make_client(URI.parse(invidious_companion_urls[Random.rand(invidious_companion_urls.size)]), + &.post(endpoint, headers: headers, body: data.to_json)) + body = response.body + if (response.status_code != 200) + raise Exception.new("status code: " + response.status_code.to_s + " and body: " + body) + end + rescue ex + raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) end else body = YT_POOL.client() do |client| @@ -660,8 +666,8 @@ module YoutubeAPI end end - if body.nil? && invidious_companion_url - raise InfoException.new("Unable to communicate with Invidious companion.") + if body.nil? && CONFIG.invidious_companion + raise InfoException.new("Error while communicating with Invidious companion: no response data.") end # Convert result to Hash From 999fcc9248c88836b50ed150e7e73ddfb4e45af0 Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Sun, 20 Oct 2024 23:53:08 +0200 Subject: [PATCH 3/5] fix Shadowing outer local variable `response` --- src/invidious/yt_backend/youtube_api.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 333d8ab7..571ca30b 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -649,11 +649,11 @@ module YoutubeAPI puts "invidious companion section" puts invidious_companion_urls[Random.rand(invidious_companion_urls.size)] begin - response = make_client(URI.parse(invidious_companion_urls[Random.rand(invidious_companion_urls.size)]), + invidious_companion_response = make_client(URI.parse(invidious_companion_urls[Random.rand(invidious_companion_urls.size)]), &.post(endpoint, headers: headers, body: data.to_json)) - body = response.body - if (response.status_code != 200) - raise Exception.new("status code: " + response.status_code.to_s + " and body: " + body) + body = invidious_companion_response.body + if (invidious_companion_response.status_code != 200) + raise Exception.new("status code: " + invidious_companion_response.status_code.to_s + " and body: " + body) end rescue ex raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) From c3d328bbe85d880568cdd933d6a2b3bb470f4f4b Mon Sep 17 00:00:00 2001 From: Emilien Devos <4016501+unixfox@users.noreply.github.com> Date: Mon, 21 Oct 2024 01:20:16 +0200 Subject: [PATCH 4/5] fixing condition for Content-Security-Policy --- src/invidious/routes/watch.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 961d7b31..2cf8a725 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -190,7 +190,7 @@ module Invidious::Routes::Watch captions: video.captions ) - if (CONFIG.invidious_companion && env.params.query["local"] == true) + if (CONFIG.invidious_companion && (preferences.local || preferences.quality == "dash")) env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] .gsub("media-src", "media-src " + video.invidious_companion["baseUrl"].as_s) From a63fca866f6046bfdc11703be8a5b150f539d2c6 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Fri, 1 Nov 2024 21:30:58 +0100 Subject: [PATCH 5/5] throw error if inv_sig_helper and invidious_companion used same time --- src/invidious.cr | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/invidious.cr b/src/invidious.cr index 56aca802..c201ef84 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -158,6 +158,12 @@ Invidious::Database.check_integrity(CONFIG) {% puts "\nDone checking player dependencies, now compiling Invidious...\n" %} {% end %} +# invidious_companion and signature_server can't work together +if CONFIG.signature_server && CONFIG.invidious_companion + puts "You can not run inv_sig_helper and invidious_companion at the same time." + exit(1) +end + # Misc DECRYPT_FUNCTION =