Compare commits

...

6 commits

Author SHA1 Message Date
Samantaz Fox
4b9f987ffb
Another attempt 2024-10-11 18:23:24 +02:00
Samantaz Fox
2f3266cd01
Attempt to fix playback errors 2024-10-11 17:17:20 +02:00
Samantaz Fox
46041f5b60
Videos: Fix missing host parameter on playback URLs when local=true 2024-10-11 17:15:33 +02:00
47237d5718
fixup! CI: Experimental branches for testing builds 2024-10-10 18:49:47 -03:00
7ba95e32b5
Try to use iOS client.
Signed-off-by: Fijxu <fijxu@nadeko.net>
2024-10-10 18:47:15 -03:00
8a3fd5751a
Revert "Feat: User supplied po_token and visitor_data"
This reverts commit b3a8866022.
2024-10-10 18:21:34 -03:00
14 changed files with 106 additions and 136 deletions

View file

@ -34,8 +34,8 @@ jobs:
with:
images: git.nadeko.net/fijxu/invidious
tags: |
type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-experimental-,enable=${{ github.ref == format('refs/heads/{0}', 'experimental') }}
type=raw,value=latest-experimental,enable=${{ github.ref == format('refs/heads/{0}', 'experimental') }}
- uses: https://code.forgejo.org/docker/build-push-action@v5
name: Build images

View file

@ -45,8 +45,6 @@ struct ConfigPreferences
property vr_mode : Bool = true
property show_nick : Bool = true
property save_player_pos : Bool = false
property po_token : String = ""
property visitor_data : String = ""
def to_tuple
{% begin %}
@ -89,8 +87,6 @@ class Config
# Used to tell Invidious it is behind a proxy, so links to resources should be https://
property https_only : Bool?
# Enable or disable CSP
property csp : Bool? = true
# HMAC signing key for CSRF tokens and verifying pubsub subscriptions
property hmac_key : String = ""
# Domain to be used for links to resources on the site where an absolute URL is required

View file

@ -4,6 +4,17 @@ module Invidious::HttpServer
module Utils
extend self
@@proxy_alive : Bool = false
def check_external_proxy
begin
response = HTTP::Client.get("#{CONFIG.external_videoplayback_proxy}")
@@proxy_alive = response.status_code == 200
rescue
@@proxy_alive = false
end
end
def proxy_video_url(raw_url : String, *, region : String? = nil, absolute : Bool = false)
url = URI.parse(raw_url)
@ -14,7 +25,11 @@ module Invidious::HttpServer
url.query_params = params
if absolute
return "#{HOST_URL}#{url.request_target}"
if @@proxy_alive
return "#{CONFIG.external_videoplayback_proxy}#{url.request_target}"
else
return "#{HOST_URL}#{url.request_target}"
end
else
return url.request_target
end

View file

@ -4,7 +4,7 @@ class Invidious::Jobs::CheckExternalProxy < Invidious::Jobs::BaseJob
def begin
loop do
Invidious::Routes::API::Manifest.check_external_proxy
HttpServer::Utils.check_external_proxy
LOGGER.info("CheckExternalProxy: Done, sleeping for 1 minute")
sleep 1.minutes
Fiber.yield

View file

@ -1,15 +1,4 @@
module Invidious::Routes::API::Manifest
@@proxy_alive : Bool = false
def self.check_external_proxy
begin
response = HTTP::Client.get("#{CONFIG.external_videoplayback_proxy}")
@@proxy_alive = response.status_code == 200
rescue
@@proxy_alive = false
end
end
# /api/manifest/dash/id/:id
def self.get_dash_video_id(env)
env.response.headers.add("Access-Control-Allow-Origin", "*")
@ -38,36 +27,21 @@ module Invidious::Routes::API::Manifest
haltf env, status_code: response.status_code
end
manifest = response.body
manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
url = baseurl.lchop("<BaseURL>")
url = url.rchop("</BaseURL>")
if local
uri = URI.parse(url)
if @@proxy_alive
url = "#{CONFIG.external_videoplayback_proxy}#{uri.request_target}host/#{uri.host}/"
else
url = "#{HOST_URL}#{uri.request_target}host/#{uri.host}/"
end
end
# Proxy URLs for video playback on invidious.
# Other API clients can get the original URLs by omiting `local=true`.
manifest = response.body.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
url = baseurl.lchop("<BaseURL>").rchop("</BaseURL>")
url = HttpServer::Utils.proxy_video_url(url, absolute: true) if local
"<BaseURL>#{url}</BaseURL>"
end
return manifest
end
adaptive_fmts = video.adaptive_fmts
# Ditto, only proxify URLs if `local=true` is used
if local
adaptive_fmts.each do |fmt|
if @@proxy_alive
fmt["url"] = JSON::Any.new("#{CONFIG.external_videoplayback_proxy}#{URI.parse(fmt["url"].as_s).request_target}")
else
fmt["url"] = JSON::Any.new("#{HOST_URL}#{URI.parse(fmt["url"].as_s).request_target}")
end
video.adaptive_fmts.each do |fmt|
fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s, absolute: true))
end
end
@ -203,8 +177,9 @@ module Invidious::Routes::API::Manifest
manifest = response.body
if local
manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
path = URI.parse(match).path
manifest = manifest.gsub(/^https:\/\/[^\n"]*/m) do |match|
uri = URI.parse(match)
path = uri.path
path = path.lchop("/videoplayback/")
path = path.rchop("/")
@ -233,7 +208,7 @@ module Invidious::Routes::API::Manifest
raw_params["fvip"] = fvip["fvip"]
end
raw_params["local"] = "true"
raw_params["host"] = uri.host.not_nil!
"#{HOST_URL}/videoplayback?#{raw_params}"
end

View file

@ -157,10 +157,12 @@ module Invidious::Routes::Embed
adaptive_fmts = video.adaptive_fmts
if params.local
fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
end
# Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
video_streams = video.video_streams
audio_streams = video.audio_streams

View file

@ -86,12 +86,6 @@ module Invidious::Routes::PreferencesRoute
show_nick ||= "off"
show_nick = show_nick == "on"
po_token = env.params.body["po_token"]?.try &.as(String)
po_token ||= CONFIG.default_user_preferences.po_token
visitor_data = env.params.body["visitor_data"]?.try &.as(String)
visitor_data ||= CONFIG.default_user_preferences.visitor_data
comments = [] of String
2.times do |i|
comments << (env.params.body["comments[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.comments[i])
@ -186,8 +180,6 @@ module Invidious::Routes::PreferencesRoute
vr_mode: vr_mode,
show_nick: show_nick,
save_player_pos: save_player_pos,
po_token: po_token,
visitor_data: visitor_data,
}.to_json)
if user = env.get? "user"

View file

@ -3,9 +3,6 @@
module Invidious::Routes::Watch
def self.handle(env)
locale = env.get("preferences").as(Preferences).locale
user_po_token = env.get("preferences").as(Preferences).po_token
user_visitor_data = env.get("preferences").as(Preferences).visitor_data
region = env.params.query["region"]?
if env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+")
@ -55,7 +52,7 @@ module Invidious::Routes::Watch
env.params.query.delete_all("listen")
begin
video = get_video(id, region: params.region, po_token: user_po_token, visitor_data: user_visitor_data)
video = get_video(id, region: params.region)
rescue ex : NotFoundException
LOGGER.error("get_video not found: #{id} : #{ex.message}")
return error_template(404, ex)
@ -131,10 +128,12 @@ module Invidious::Routes::Watch
end
if params.local
fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
end
# Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
video_streams = video.video_streams
audio_streams = video.audio_streams

View file

@ -57,10 +57,6 @@ struct Preferences
property volume : Int32 = CONFIG.default_user_preferences.volume
property save_player_pos : Bool = CONFIG.default_user_preferences.save_player_pos
@[YAML::Field(converter: Preferences::ProcessString)]
property po_token : String = ""
property visitor_data : String = ""
module BoolToString
def self.to_json(value : String, json : JSON::Builder)
json.string value

View file

@ -294,7 +294,7 @@ struct Video
predicate_bool upcoming, isUpcoming
end
def get_video(id, refresh = true, region = nil, force_refresh = false, po_token = "", visitor_data = nil)
def get_video(id, refresh = true, region = nil, force_refresh = false)
if (video = Invidious::Database::Videos.select(id)) && !region
# If record was last updated over 10 minutes ago, or video has since premiered,
# refresh (expire param in response lasts for 6 hours)
@ -304,7 +304,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false, po_token
force_refresh ||
video.schema_version != Video::SCHEMA_VERSION # cache control
begin
video = fetch_video(id, region, po_token, visitor_data)
video = fetch_video(id, region)
Invidious::Database::Videos.insert(video)
rescue ex
Invidious::Database::Videos.delete(id)
@ -312,7 +312,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false, po_token
end
end
else
video = fetch_video(id, region, po_token, visitor_data)
video = fetch_video(id, region)
Invidious::Database::Videos.insert(video) if !region
end
@ -320,11 +320,11 @@ def get_video(id, refresh = true, region = nil, force_refresh = false, po_token
rescue DB::Error
# Avoid common `DB::PoolRetryAttemptsExceeded` error and friends
# Note: All DB errors inherit from `DB::Error`
return fetch_video(id, region, po_token, visitor_data)
return fetch_video(id, region)
end
def fetch_video(id, region, po_token, visitor_data)
info = extract_video_info(video_id: id, po_token: po_token, visitor_data: visitor_data)
def fetch_video(id, region)
info = extract_video_info(video_id: id, level: 0)
if reason = info["reason"]?
if reason == "Video unavailable"

View file

@ -50,13 +50,22 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
}
end
def extract_video_info(video_id : String, po_token, visitor_data)
def extract_video_info(video_id : String, *, level = 0, client_type = YoutubeAPI::ClientType::WebMobileT2)
# Infinite recursion prevention
level += 1
if level >= 3
return {
"version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
"reason" => JSON::Any.new("All counter-measures exhausted"),
}
end
# Init client config for the API
client_config = YoutubeAPI::ClientConfig.new
client_config.client_type = client_type
# Fetch data from the player endpoint
player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config, po_token: po_token, visitor_data: visitor_data)
player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
if playability_status != "OK"
@ -65,10 +74,15 @@ def extract_video_info(video_id : String, po_token, visitor_data)
reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
reason ||= player_response.dig("playabilityStatus", "reason").as_s
# Stop here if video is not a scheduled livestream or
# for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help
if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) ||
# Show the playability status in the reason message
reason = "#{reason} (#{playability_status})"
if (playability_status == "UNPLAYABLE" && reason.includes?("Get the YouTube app")) || reason.includes?("protect our community")
return extract_video_info(video_id: video_id, level: level, client_type: YoutubeAPI::ClientType::IOS)
elsif !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) ||
playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails")
# Stop here if video is not a scheduled livestream or
# for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help
return {
"version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
"reason" => JSON::Any.new(reason),
@ -92,7 +106,7 @@ def extract_video_info(video_id : String, po_token, visitor_data)
end
# Don't fetch the next endpoint if the video is unavailable.
if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED", "UNPLAYABLE"}.any?(playability_status)
next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
player_response = player_response.merge(next_response)
end
@ -102,24 +116,14 @@ def extract_video_info(video_id : String, po_token, visitor_data)
new_player_response = nil
# Don't use Android client if po_token is passed because po_token doesn't
# work for Android client.
if reason.nil? && CONFIG.po_token.nil?
# Fetch the video streams using an Android client in order to get the
# decrypted URLs and maybe fix throttling issues (#2194). See the
# following issue for an explanation about decrypted URLs:
# https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data)
if reason || DECRYPT_FUNCTION.nil? || CONFIG.po_token.nil?
client_config.client_type = YoutubeAPI::ClientType::IOS
new_player_response = try_fetch_streaming_data(video_id, client_config)
end
# Last hope
# Only trigger if reason found and po_token or didn't work wth Android client.
# TvHtml5ScreenEmbed now requires sig helper for it to work but po_token is not required
# if the IP address is not blocked.
if CONFIG.po_token && reason || CONFIG.po_token.nil? && new_player_response.nil?
if reason && !DECRYPT_FUNCTION.nil? && CONFIG.po_token.nil?
client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data)
new_player_response = try_fetch_streaming_data(video_id, client_config)
end
# Replace player response and reset reason
@ -140,7 +144,7 @@ def extract_video_info(video_id : String, po_token, visitor_data)
if streaming_data = player_response["streamingData"]?
%w[formats adaptiveFormats].each do |key|
streaming_data.as_h[key]?.try &.as_a.each do |format|
format.as_h["url"] = JSON::Any.new(convert_url(format, po_token))
format.as_h["url"] = JSON::Any.new(convert_url(format))
end
end
@ -153,9 +157,9 @@ def extract_video_info(video_id : String, po_token, visitor_data)
return params
end
def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig, po_token, visitor_data) : Hash(String, JSON::Any)?
def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config, po_token: po_token, visitor_data: visitor_data)
response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)
playability_status = response["playabilityStatus"]["status"]
LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
@ -455,7 +459,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
return params
end
private def convert_url(fmt, po_token)
private def convert_url(fmt)
if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
sp = cfr["sp"]
url = URI.parse(cfr["url"])
@ -470,13 +474,15 @@ private def convert_url(fmt, po_token)
params = url.query_params
end
n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
params["n"] = n if n
if old_n = params["n"]?
n = DECRYPT_FUNCTION.try &.decrypt_nsig(old_n)
params["n"] = n if n
end
if !po_token.empty?
params["pot"] = po_token
elsif token = CONFIG.po_token
params["pot"] = token
if token = CONFIG.po_token
if {"WEB", "TVHTML5"}.any? { |x| params["c"].starts_with?(x) }
params["pot"] = token
end
end
url.query_params = params

View file

@ -4,9 +4,6 @@
<% if params.autoplay %>autoplay<% end %>
<% if params.video_loop %>loop<% end %>
<% if params.controls %>controls<% end %>>
<% if (hlsvp = video.hls_manifest_url) && !CONFIG.disabled?("livestreams") %>
<source src="<%= URI.parse(hlsvp).request_target %><% if params.local %>?local=true<% end %>" type="application/x-mpegURL" label="livestream">
<% else %>
<% if params.listen %>
<% # default to 128k m4a stream
best_m4a_stream_index = 0
@ -57,6 +54,11 @@
<% end %>
<% end %>
<% if (hlsvp = video.hls_manifest_url) && !CONFIG.disabled?("livestreams") %>
<source src="<%= URI.parse(hlsvp).request_target %><% if params.local %>?local=true<% end %>" type="application/x-mpegURL" label="livestream">
<% end %>
<% preferred_captions.each do |caption| %>
<track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name %>" label="<%= caption.name %>">
<% end %>
@ -64,7 +66,6 @@
<% captions.each do |caption| %>
<track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name %>" label="<%= caption.name %>">
<% end %>
<% end %>
</video>
<script id="player_data" type="application/json">

View file

@ -126,16 +126,6 @@
<input name="save_player_pos" id="save_player_pos" type="checkbox" <% if preferences.save_player_pos %>checked<% end %>>
</div>
<div class="pure-control-group">
<label for="po_token"><%= translate(locale, "preferences_po_token") %></label>
<input name="po_token" id="po_token" type="text" value="<%= preferences.po_token %>">
</div>
<div class="pure-control-group">
<label for="visitor_data"><%= translate(locale, "preferences_visitor_data") %></label>
<input name="visitor_data" id="visitor_data" type="text" value="<%= preferences.visitor_data %>">
</div>
<legend><%= translate(locale, "preferences_category_visual") %></legend>
<div class="pure-control-group">

View file

@ -17,7 +17,7 @@ module YoutubeAPI
# For Apple device names, see https://gist.github.com/adamawolf/3048717
# For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases,
# then go to the dedicated article of the major version you want.
private IOS_APP_VERSION = "19.32.8"
private IOS_APP_VERSION = "19.40.4"
private IOS_USER_AGENT = "com.google.ios.youtube/#{IOS_APP_VERSION} (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)"
private IOS_VERSION = "17.6.1.21G93" # Major.Minor.Patch.Build
@ -28,6 +28,7 @@ module YoutubeAPI
Web
WebEmbeddedPlayer
WebMobile
WebMobileT2
WebScreenEmbed
Android
@ -48,7 +49,7 @@ module YoutubeAPI
ClientType::Web => {
name: "WEB",
name_proto: "1",
version: "2.20240814.00.00",
version: "2.20241010.01.00",
screen: "WATCH_FULL_SCREEN",
os_name: "Windows",
os_version: WINDOWS_VERSION,
@ -66,11 +67,19 @@ module YoutubeAPI
ClientType::WebMobile => {
name: "MWEB",
name_proto: "2",
version: "2.20240813.02.00",
version: "2.20241010.02.00",
os_name: "Android",
os_version: ANDROID_VERSION,
platform: "MOBILE",
},
ClientType::WebMobileT2 => {
name: "MWEB_TIER_2",
name_proto: "27",
version: "9.20241010",
os_name: "iPhone",
os_version: IOS_VERSION,
platform: "MOBILE",
},
ClientType::WebScreenEmbed => {
name: "WEB",
name_proto: "1",
@ -198,8 +207,6 @@ module YoutubeAPI
# (this is passed as the `gl` parameter).
property region : String | Nil
@@visitor_data : String | Nil
# Initialization function
def initialize(
*,
@ -322,9 +329,7 @@ module YoutubeAPI
client_context["client"]["platform"] = platform
end
if !@@visitor_data.not_nil!.empty?
client_context["client"]["visitorData"] = @@visitor_data.not_nil!
elsif CONFIG.visitor_data.is_a?(String)
if CONFIG.visitor_data.is_a?(String)
client_context["client"]["visitorData"] = CONFIG.visitor_data.as(String)
end
@ -459,13 +464,8 @@ module YoutubeAPI
video_id : String,
*, # Force the following parameters to be passed by name
params : String,
client_config : ClientConfig | Nil = nil,
po_token : String,
visitor_data : String | Nil
client_config : ClientConfig | Nil = nil
)
if visitor_data
@@visitor_data = visitor_data
end
# Playback context, separate because it can be different between clients
playback_ctx = {
"html5Preference" => "HTML5_PREF_WANTS",
@ -491,7 +491,7 @@ module YoutubeAPI
"contentPlaybackContext" => playback_ctx,
},
"serviceIntegrityDimensions" => {
"poToken" => po_token || CONFIG.po_token,
"poToken" => CONFIG.po_token,
},
}
@ -605,7 +605,7 @@ module YoutubeAPI
def _post_json(
endpoint : String,
data : Hash,
client_config : ClientConfig | Nil,
client_config : ClientConfig | Nil
) : Hash(String, JSON::Any)
# Use the default client config if nil is passed
client_config ||= DEFAULT_CLIENT_CONFIG
@ -625,9 +625,7 @@ module YoutubeAPI
headers["User-Agent"] = user_agent
end
if !@@visitor_data.not_nil!.empty?
headers["X-Goog-Visitor-Id"] = @@visitor_data.not_nil!
elsif CONFIG.visitor_data.is_a?(String)
if CONFIG.visitor_data.is_a?(String)
headers["X-Goog-Visitor-Id"] = CONFIG.visitor_data.as(String)
end