From b3a88660222e82c93fca18cdfb0ba5a5d515801f Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 10 Oct 2024 17:43:18 -0300 Subject: [PATCH] Feat: User supplied po_token and visitor_data --- src/invidious/config.cr | 4 ++++ src/invidious/routes/preferences.cr | 8 ++++++++ src/invidious/routes/watch.cr | 5 ++++- src/invidious/user/preferences.cr | 4 ++++ src/invidious/videos.cr | 12 ++++++------ src/invidious/videos/parser.cr | 20 +++++++++++--------- src/invidious/views/user/preferences.ecr | 10 ++++++++++ src/invidious/yt_backend/youtube_api.cr | 21 ++++++++++++++++----- 8 files changed, 63 insertions(+), 21 deletions(-) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 5c1cf578..b1e10684 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -45,6 +45,8 @@ struct ConfigPreferences property vr_mode : Bool = true property show_nick : Bool = true property save_player_pos : Bool = false + property po_token : String = "" + property visitor_data : String = "" def to_tuple {% begin %} @@ -87,6 +89,8 @@ class Config # Used to tell Invidious it is behind a proxy, so links to resources should be https:// property https_only : Bool? + # Enable or disable CSP + property csp : Bool? = true # HMAC signing key for CSRF tokens and verifying pubsub subscriptions property hmac_key : String = "" # Domain to be used for links to resources on the site where an absolute URL is required diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 5d3dd9b7..09069486 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -86,6 +86,12 @@ module Invidious::Routes::PreferencesRoute show_nick ||= "off" show_nick = show_nick == "on" + po_token = env.params.body["po_token"]?.try &.as(String) + po_token ||= CONFIG.default_user_preferences.po_token + + visitor_data = env.params.body["visitor_data"]?.try &.as(String) + visitor_data ||= CONFIG.default_user_preferences.visitor_data + comments = [] of String 2.times do |i| comments << (env.params.body["comments[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.comments[i]) @@ -180,6 +186,8 @@ module Invidious::Routes::PreferencesRoute vr_mode: vr_mode, show_nick: show_nick, save_player_pos: save_player_pos, + po_token: po_token, + visitor_data: visitor_data, }.to_json) if user = env.get? "user" diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 612a689f..7e8a630f 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -3,6 +3,9 @@ module Invidious::Routes::Watch def self.handle(env) locale = env.get("preferences").as(Preferences).locale + user_po_token = env.get("preferences").as(Preferences).po_token + user_visitor_data = env.get("preferences").as(Preferences).visitor_data + region = env.params.query["region"]? if env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+") @@ -52,7 +55,7 @@ module Invidious::Routes::Watch env.params.query.delete_all("listen") begin - video = get_video(id, region: params.region) + video = get_video(id, region: params.region, po_token: user_po_token, visitor_data: user_visitor_data) rescue ex : NotFoundException LOGGER.error("get_video not found: #{id} : #{ex.message}") return error_template(404, ex) diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr index 0a8525f3..342d91b3 100644 --- a/src/invidious/user/preferences.cr +++ b/src/invidious/user/preferences.cr @@ -57,6 +57,10 @@ struct Preferences property volume : Int32 = CONFIG.default_user_preferences.volume property save_player_pos : Bool = CONFIG.default_user_preferences.save_player_pos + @[YAML::Field(converter: Preferences::ProcessString)] + property po_token : String = "" + property visitor_data : String = "" + module BoolToString def self.to_json(value : String, json : JSON::Builder) json.string value diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index c6e69ee5..f1b46bf8 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -294,7 +294,7 @@ struct Video predicate_bool upcoming, isUpcoming end -def get_video(id, refresh = true, region = nil, force_refresh = false) +def get_video(id, refresh = true, region = nil, force_refresh = false, po_token = "", visitor_data = nil) if (video = Invidious::Database::Videos.select(id)) && !region # If record was last updated over 10 minutes ago, or video has since premiered, # refresh (expire param in response lasts for 6 hours) @@ -304,7 +304,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) force_refresh || video.schema_version != Video::SCHEMA_VERSION # cache control begin - video = fetch_video(id, region) + video = fetch_video(id, region, po_token, visitor_data) Invidious::Database::Videos.insert(video) rescue ex Invidious::Database::Videos.delete(id) @@ -312,7 +312,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) end end else - video = fetch_video(id, region) + video = fetch_video(id, region, po_token, visitor_data) Invidious::Database::Videos.insert(video) if !region end @@ -320,11 +320,11 @@ def get_video(id, refresh = true, region = nil, force_refresh = false) rescue DB::Error # Avoid common `DB::PoolRetryAttemptsExceeded` error and friends # Note: All DB errors inherit from `DB::Error` - return fetch_video(id, region) + return fetch_video(id, region, po_token, visitor_data) end -def fetch_video(id, region) - info = extract_video_info(video_id: id) +def fetch_video(id, region, po_token, visitor_data) + info = extract_video_info(video_id: id, po_token: po_token, visitor_data: visitor_data) if reason = info["reason"]? if reason == "Video unavailable" diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 4683058b..ad891dc4 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -50,12 +50,12 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? } end -def extract_video_info(video_id : String) +def extract_video_info(video_id : String, po_token, visitor_data) # Init client config for the API client_config = YoutubeAPI::ClientConfig.new # Fetch data from the player endpoint - player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config) + player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config, po_token: po_token, visitor_data: visitor_data) playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s @@ -110,7 +110,7 @@ def extract_video_info(video_id : String) # following issue for an explanation about decrypted URLs: # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite - new_player_response = try_fetch_streaming_data(video_id, client_config) + new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data) end # Last hope @@ -119,7 +119,7 @@ def extract_video_info(video_id : String) # if the IP address is not blocked. if CONFIG.po_token && reason || CONFIG.po_token.nil? && new_player_response.nil? client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed - new_player_response = try_fetch_streaming_data(video_id, client_config) + new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data) end # Replace player response and reset reason @@ -140,7 +140,7 @@ def extract_video_info(video_id : String) if streaming_data = player_response["streamingData"]? %w[formats adaptiveFormats].each do |key| streaming_data.as_h[key]?.try &.as_a.each do |format| - format.as_h["url"] = JSON::Any.new(convert_url(format)) + format.as_h["url"] = JSON::Any.new(convert_url(format, po_token)) end end @@ -153,9 +153,9 @@ def extract_video_info(video_id : String) return params end -def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? +def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig, po_token, visitor_data) : Hash(String, JSON::Any)? LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") - response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config) + response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config, po_token: po_token, visitor_data: visitor_data) playability_status = response["playabilityStatus"]["status"] LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") @@ -455,7 +455,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any return params end -private def convert_url(fmt) +private def convert_url(fmt, po_token) if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) } sp = cfr["sp"] url = URI.parse(cfr["url"]) @@ -473,7 +473,9 @@ private def convert_url(fmt) n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"]) params["n"] = n if n - if token = CONFIG.po_token + if !po_token.empty? + params["pot"] = po_token + elsif token = CONFIG.po_token params["pot"] = token end diff --git a/src/invidious/views/user/preferences.ecr b/src/invidious/views/user/preferences.ecr index 9dbe298c..fcde2313 100644 --- a/src/invidious/views/user/preferences.ecr +++ b/src/invidious/views/user/preferences.ecr @@ -126,6 +126,16 @@ checked<% end %>> +
+ + +
+ +
+ + +
+ <%= translate(locale, "preferences_category_visual") %>
diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index d66bf7aa..8602cc2d 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -198,6 +198,8 @@ module YoutubeAPI # (this is passed as the `gl` parameter). property region : String | Nil + @@visitor_data : String | Nil + # Initialization function def initialize( *, @@ -320,7 +322,9 @@ module YoutubeAPI client_context["client"]["platform"] = platform end - if CONFIG.visitor_data.is_a?(String) + if !@@visitor_data.not_nil!.empty? + client_context["client"]["visitorData"] = @@visitor_data.not_nil! + elsif CONFIG.visitor_data.is_a?(String) client_context["client"]["visitorData"] = CONFIG.visitor_data.as(String) end @@ -455,8 +459,13 @@ module YoutubeAPI video_id : String, *, # Force the following parameters to be passed by name params : String, - client_config : ClientConfig | Nil = nil + client_config : ClientConfig | Nil = nil, + po_token : String, + visitor_data : String | Nil ) + if visitor_data + @@visitor_data = visitor_data + end # Playback context, separate because it can be different between clients playback_ctx = { "html5Preference" => "HTML5_PREF_WANTS", @@ -482,7 +491,7 @@ module YoutubeAPI "contentPlaybackContext" => playback_ctx, }, "serviceIntegrityDimensions" => { - "poToken" => CONFIG.po_token, + "poToken" => po_token || CONFIG.po_token, }, } @@ -596,7 +605,7 @@ module YoutubeAPI def _post_json( endpoint : String, data : Hash, - client_config : ClientConfig | Nil + client_config : ClientConfig | Nil, ) : Hash(String, JSON::Any) # Use the default client config if nil is passed client_config ||= DEFAULT_CLIENT_CONFIG @@ -616,7 +625,9 @@ module YoutubeAPI headers["User-Agent"] = user_agent end - if CONFIG.visitor_data.is_a?(String) + if !@@visitor_data.not_nil!.empty? + headers["X-Goog-Visitor-Id"] = @@visitor_data.not_nil! + elsif CONFIG.visitor_data.is_a?(String) headers["X-Goog-Visitor-Id"] = CONFIG.visitor_data.as(String) end