Merge remote-tracking branch 'upstream/master' into master
All checks were successful
Invidious CI / build (push) Successful in 5m35s

This commit is contained in:
Fijxu 2024-10-31 20:22:59 -03:00
commit e2276ace1b
Signed by: Fijxu
GPG key ID: 32C1DDF333EDA6A4
17 changed files with 202 additions and 124 deletions

View file

@ -38,10 +38,11 @@ jobs:
matrix:
stable: [true]
crystal:
- 1.9.2
- 1.10.1
- 1.11.2
- 1.12.1
- 1.13.2
- 1.14.0
include:
- crystal: nightly
stable: false
@ -51,6 +52,11 @@ jobs:
with:
submodules: true
- name: Install required APT packages
run: |
sudo apt install -y libsqlite3-dev
shell: bash
- name: Install Crystal
uses: crystal-lang/install-crystal@v1.8.0
with:

View file

@ -5,6 +5,12 @@
### Full list of pull requests merged since the last release (newest first)
* Add "Filipino (auto-generated)" to the list of caption languages ([#4995], by @SamantazFox)
* Makefile: Add MT option to enable the 'preview_mt' flag ([#4993], by @SamantazFox)
* SigHelper: Reconnect to signature helper ([#4991], thanks @Fijxu)
* Fix player menus hiding onHover ready ([#4750], thanks @giacomocerquone)
* Use connection pools when requesting images from YouTube ([#4326], thanks @syeopite)
* Add support for using Invidious through a HTTP Proxy ([#4270], thanks @syeopite)
* Search: Fix 'youtu.be' URLs in sanitizer ([#4894], by @SamantazFox)
* Ameba: Disable Style/RedundantNext rule ([#4888], thanks @syeopite)
* Playlists: Fix 'invalid byte sequence' error when subscribing ([#4887], thanks @DmitrySandalov)
@ -22,7 +28,10 @@
[#4122]: https://github.com/iv-org/invidious/pull/4122
[#4193]: https://github.com/iv-org/invidious/pull/4193
[#4270]: https://github.com/iv-org/invidious/pull/4270
[#4326]: https://github.com/iv-org/invidious/pull/4326
[#4652]: https://github.com/iv-org/invidious/pull/4652
[#4750]: https://github.com/iv-org/invidious/pull/4750
[#4850]: https://github.com/iv-org/invidious/pull/4850
[#4862]: https://github.com/iv-org/invidious/pull/4862
[#4863]: https://github.com/iv-org/invidious/pull/4863
@ -33,6 +42,9 @@
[#4928]: https://github.com/iv-org/invidious/pull/4928
[#4930]: https://github.com/iv-org/invidious/pull/4930
[#4942]: https://github.com/iv-org/invidious/pull/4942
[#4991]: https://github.com/iv-org/invidious/pull/4991
[#4993]: https://github.com/iv-org/invidious/pull/4993
[#4995]: https://github.com/iv-org/invidious/pull/4995
## v2.20240825.2 (2024-08-26)

View file

@ -7,6 +7,11 @@ STATIC := 0
NO_DBG_SYMBOLS := 0
# Enable multi-threading.
# Warning: Experimental feature!!
# invidious is not stable when MT is enabled.
MT := 0
FLAGS ?=
@ -19,6 +24,10 @@ ifeq ($(STATIC), 1)
FLAGS += --static
endif
ifeq ($(MT), 1)
FLAGS += -Dpreview_mt
endif
ifeq ($(NO_DBG_SYMBOLS), 1)
FLAGS += --no-debug

View file

@ -68,6 +68,7 @@
.video-js.player-style-youtube .vjs-menu-button-popup .vjs-menu {
margin-bottom: 2em;
padding-top: 2em
}
.video-js.player-style-youtube .vjs-progress-control .vjs-progress-holder, .video-js.player-style-youtube .vjs-progress-control {height: 5px;

View file

@ -173,6 +173,17 @@ https_only: false
##
#force_resolve:
##
## Configuration for using a HTTP proxy
##
## If unset, then no HTTP proxy will be used.
##
http_proxy:
user:
password:
host:
port:
##
## Use Innertube's transcripts API instead of timedtext for closed captions

View file

@ -287,6 +287,7 @@
"Esperanto": "Esperanto",
"Estonian": "Estonian",
"Filipino": "Filipino",
"Filipino (auto-generated)": "Filipino (auto-generated)",
"Finnish": "Finnish",
"French": "French",
"French (auto-generated)": "French (auto-generated)",

2
mocks

@ -1 +1 @@
Subproject commit 11ec372f72747c09d48ffef04843f72be67d5b54
Subproject commit b55d58dea94f7144ff0205857dfa70ec14eaa872

View file

@ -10,7 +10,7 @@ shards:
backtracer:
git: https://github.com/sija/backtracer.cr.git
version: 1.2.1
version: 1.2.2
db:
git: https://github.com/crystal-lang/crystal-db.git
@ -23,6 +23,9 @@ shards:
inotify:
git: https://github.com/petoem/inotify.cr.git
version: 1.0.3
http_proxy:
git: https://github.com/mamantoha/http_proxy.git
version: 0.10.3
kemal:
git: https://github.com/kemalcr/kemal.git
@ -54,7 +57,7 @@ shards:
spectator:
git: https://github.com/icy-arctic-fox/spectator.git
version: 0.10.4
version: 0.10.6
sqlite3:
git: https://github.com/crystal-lang/crystal-sqlite3.git

View file

@ -33,6 +33,9 @@ dependencies:
inotify:
github: petoem/inotify.cr
version: 1.0.3
http_proxy:
github: mamantoha/http_proxy
version: ~> 0.10.3
development_dependencies:
spectator:

View file

@ -17,8 +17,8 @@ Spectator.describe "parse_video_info" do
# Basic video infos
expect(info["title"].as_s).to eq("I Gave My 100,000,000th Subscriber An Island")
expect(info["views"].as_i).to eq(126_573_823)
expect(info["likes"].as_i).to eq(5_157_654)
expect(info["views"].as_i).to eq(220_226_287)
expect(info["likes"].as_i).to eq(6_870_691)
# For some reason the video length from VideoDetails and the
# one from microformat differs by 1s...
@ -48,12 +48,12 @@ Spectator.describe "parse_video_info" do
expect(info["relatedVideos"].as_a.size).to eq(20)
expect(info["relatedVideos"][0]["id"]).to eq("Hwybp38GnZw")
expect(info["relatedVideos"][0]["title"]).to eq("I Built Willy Wonka's Chocolate Factory!")
expect(info["relatedVideos"][0]["id"]).to eq("krsBRQbOPQ4")
expect(info["relatedVideos"][0]["title"]).to eq("$1 vs $250,000,000 Private Island!")
expect(info["relatedVideos"][0]["author"]).to eq("MrBeast")
expect(info["relatedVideos"][0]["ucid"]).to eq("UCX6OQ3DkcsbYNE6H8uQQuVA")
expect(info["relatedVideos"][0]["view_count"]).to eq("179877630")
expect(info["relatedVideos"][0]["short_view_count"]).to eq("179M")
expect(info["relatedVideos"][0]["view_count"]).to eq("230617484")
expect(info["relatedVideos"][0]["short_view_count"]).to eq("230M")
expect(info["relatedVideos"][0]["author_verified"]).to eq("true")
# Description
@ -76,11 +76,11 @@ Spectator.describe "parse_video_info" do
expect(info["ucid"].as_s).to eq("UCX6OQ3DkcsbYNE6H8uQQuVA")
expect(info["authorThumbnail"].as_s).to eq(
"https://yt3.ggpht.com/ytc/AL5GRJVuqw82ERvHzsmBxL7avr1dpBtsVIXcEzBPZaloFg=s48-c-k-c0x00ffffff-no-rj"
"https://yt3.ggpht.com/fxGKYucJAVme-Yz4fsdCroCFCrANWqw0ql4GYuvx8Uq4l_euNJHgE-w9MTkLQA805vWCi-kE0g=s48-c-k-c0x00ffffff-no-rj"
)
expect(info["authorVerified"].as_bool).to be_true
expect(info["subCountText"].as_s).to eq("143M")
expect(info["subCountText"].as_s).to eq("320M")
end
it "parses a regular video with no descrition/comments" do
@ -99,8 +99,8 @@ Spectator.describe "parse_video_info" do
# Basic video infos
expect(info["title"].as_s).to eq("Chris Rea - Auberge")
expect(info["views"].as_i).to eq(10_943_126)
expect(info["likes"].as_i).to eq(0)
expect(info["views"].as_i).to eq(14_324_584)
expect(info["likes"].as_i).to eq(35_870)
expect(info["lengthSeconds"].as_i).to eq(283_i64)
expect(info["published"].as_s).to eq("2012-05-21T00:00:00Z")
@ -132,14 +132,14 @@ Spectator.describe "parse_video_info" do
# Related videos
expect(info["relatedVideos"].as_a.size).to eq(19)
expect(info["relatedVideos"].as_a.size).to eq(20)
expect(info["relatedVideos"][0]["id"]).to eq("Ww3KeZ2_Yv4")
expect(info["relatedVideos"][0]["title"]).to eq("Chris Rea")
expect(info["relatedVideos"][0]["author"]).to eq("PanMusic")
expect(info["relatedVideos"][0]["ucid"]).to eq("UCsKAPSuh1iNbLWUga_igPyA")
expect(info["relatedVideos"][0]["view_count"]).to eq("31581")
expect(info["relatedVideos"][0]["short_view_count"]).to eq("31K")
expect(info["relatedVideos"][0]["id"]).to eq("gUUdQfnshJ4")
expect(info["relatedVideos"][0]["title"]).to eq("Chris Rea - The Road To Hell 1989 Full Version")
expect(info["relatedVideos"][0]["author"]).to eq("NEA ZIXNH")
expect(info["relatedVideos"][0]["ucid"]).to eq("UCYMEOGcvav3gCgImK2J07CQ")
expect(info["relatedVideos"][0]["view_count"]).to eq("53298661")
expect(info["relatedVideos"][0]["short_view_count"]).to eq("53M")
expect(info["relatedVideos"][0]["author_verified"]).to eq("false")
# Description
@ -156,11 +156,13 @@ Spectator.describe "parse_video_info" do
# Author infos
expect(info["author"].as_s).to eq("ChrisReaOfficial")
expect(info["author"].as_s).to eq("ChrisReaVideos")
expect(info["ucid"].as_s).to eq("UC_5q6nWPbD30-y6oiWF_oNA")
expect(info["authorThumbnail"].as_s).to be_empty
expect(info["authorThumbnail"].as_s).to eq(
"https://yt3.ggpht.com/ytc/AIdro_n71nsegpKfjeRKwn1JJmK5IVMh_7j5m_h3_1KnUUg=s48-c-k-c0x00ffffff-no-rj"
)
expect(info["authorVerified"].as_bool).to be_false
expect(info["subCountText"].as_s).to eq("-")
expect(info["subCountText"].as_s).to eq("3.11K")
end
end

View file

@ -23,6 +23,7 @@ require "kilt"
require "./ext/kemal_content_for.cr"
require "./ext/kemal_static_file_handler.cr"
require "http_proxy"
require "athena-negotiation"
require "openssl/hmac"
require "option_parser"
@ -114,6 +115,10 @@ SOFTWARE = {
YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
# Image request pool
GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size)
# CLI
Kemal.config.extra_options do |parser|
parser.banner = "Usage: invidious [arguments]"

View file

@ -57,6 +57,15 @@ struct ConfigPreferences
end
end
struct HTTPProxyConfig
include YAML::Serializable
property user : String
property password : String
property host : String
property port : Int32
end
class Config
include YAML::Serializable
@ -157,6 +166,8 @@ class Config
property host_binding : String = "0.0.0.0"
# Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
property pool_size : Int32 = 100
# HTTP Proxy configuration
property http_proxy : HTTPProxyConfig? = nil
# Use Innertube's transcripts API instead of timedtext for closed captions
property use_innertube_for_captions : Bool = false

View file

@ -18,6 +18,40 @@ end
class HTTP::Client
property family : Socket::Family = Socket::Family::UNSPEC
# Override stdlib to automatically initialize proxy if configured
#
# Accurate as of crystal 1.12.1
def initialize(@host : String, port = nil, tls : TLSContext = nil)
check_host_only(@host)
{% if flag?(:without_openssl) %}
if tls
raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time"
end
@tls = nil
{% else %}
@tls = case tls
when true
OpenSSL::SSL::Context::Client.new
when OpenSSL::SSL::Context::Client
tls
when false, nil
nil
end
{% end %}
@port = (port || (@tls ? 443 : 80)).to_i
self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
end
def initialize(@io : IO, @host = "", @port = 80)
@reconnect = false
self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
end
private def io
io = @io
return io if io

View file

@ -175,7 +175,6 @@ module Invidious::SigHelper
@queue = {} of TransactionID => Transaction
@conn : Connection
@uri_or_path : String
def initialize(@uri_or_path)
@ -201,7 +200,7 @@ module Invidious::SigHelper
@conn = Connection.new(@uri_or_path)
LOGGER.info("SigHelper: Reconnected to SigHelper!")
rescue ex
LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}' retrying")
LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying")
sleep 500.milliseconds
next
end

View file

@ -11,29 +11,9 @@ module Invidious::Routes::Images
end
end
# We're encapsulating this into a proc in order to easily reuse this
# portion of the code for each request block below.
request_proc = ->(response : HTTP::Client::Response) {
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300
env.response.headers.delete("Transfer-Encoding")
return
end
proxy_file(response, env)
}
begin
HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
return request_proc.call(resp)
GGPHT_POOL.client &.get(url, headers) do |resp|
return self.proxy_image(env, resp)
end
rescue ex
end
@ -61,27 +41,10 @@ module Invidious::Routes::Images
end
end
request_proc = ->(response : HTTP::Client::Response) {
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Connection"] = "close"
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300
return env.response.headers.delete("Transfer-Encoding")
end
proxy_file(response, env)
}
begin
HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
get_ytimg_pool(authority).client &.get(url, headers) do |resp|
env.response.headers["Connection"] = "close"
return self.proxy_image(env, resp)
end
rescue ex
end
@ -101,26 +64,9 @@ module Invidious::Routes::Images
end
end
request_proc = ->(response : HTTP::Client::Response) {
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300 && response.status_code != 404
return env.response.headers.delete("Transfer-Encoding")
end
proxy_file(response, env)
}
begin
HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
get_ytimg_pool("i9").client &.get(url, headers) do |resp|
return self.proxy_image(env, resp)
end
rescue ex
end
@ -165,8 +111,7 @@ module Invidious::Routes::Images
if name == "maxres.jpg"
build_thumbnails(id).each do |thumb|
thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
# This can likely be optimized into a (small) pool sometime in the future.
if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
if get_ytimg_pool("i9").client &.head(thumbnail_resource_path, headers).status_code == 200
name = thumb[:url] + ".jpg"
break
end
@ -181,29 +126,28 @@ module Invidious::Routes::Images
end
end
request_proc = ->(response : HTTP::Client::Response) {
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300 && response.status_code != 404
return env.response.headers.delete("Transfer-Encoding")
end
proxy_file(response, env)
}
begin
# This can likely be optimized into a (small) pool sometime in the future.
HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
return request_proc.call(resp)
get_ytimg_pool("i").client &.get(url, headers) do |resp|
return self.proxy_image(env, resp)
end
rescue ex
end
end
private def self.proxy_image(env, response)
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300
return env.response.headers.delete("Transfer-Encoding")
end
return proxy_file(response, env)
end
end

View file

@ -123,6 +123,7 @@ module Invidious::Videos
"Esperanto",
"Estonian",
"Filipino",
"Filipino (auto-generated)",
"Finnish",
"French",
"French (auto-generated)",

View file

@ -1,17 +1,6 @@
def add_yt_headers(request)
request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
request.headers["User-Agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
request.headers["Accept-Language"] ||= "en-US,en;q=0.9"
# Preserve original cookies and add new YT consent cookie for EU servers
request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}"
if !CONFIG.cookies.empty?
request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
end
end
# Mapping of subdomain => YoutubeConnectionPool
# This is needed as we may need to access arbitrary subdomains of ytimg
private YTIMG_POOLS = {} of String => YoutubeConnectionPool
struct YoutubeConnectionPool
property! url : URI
@ -26,12 +15,16 @@ struct YoutubeConnectionPool
def client(&)
conn = pool.checkout
# Proxy needs to be reinstated every time we get a client from the pool
conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
begin
response = yield conn
rescue ex
conn.close
conn = HTTP::Client.new(url)
conn = HTTP::Client.new(url)
conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
conn.family = CONFIG.force_resolve
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
@ -54,6 +47,21 @@ struct YoutubeConnectionPool
end
end
def add_yt_headers(request)
request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
request.headers["Accept-Language"] ||= "en-us,en;q=0.5"
# Preserve original cookies and add new YT consent cookie for EU servers
request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}"
if !CONFIG.cookies.empty?
request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
end
end
def make_client(url : URI, region = nil, force_resolve : Bool = false)
client = HTTP::Client.new(url)
@ -77,3 +85,31 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &)
client.close
end
end
def make_configured_http_proxy_client
# This method is only called when configuration for an HTTP proxy are set
config_proxy = CONFIG.http_proxy.not_nil!
return HTTP::Proxy::Client.new(
config_proxy.host,
config_proxy.port,
username: config_proxy.user,
password: config_proxy.password,
)
end
# Fetches a HTTP pool for the specified subdomain of ytimg.com
#
# Creates a new one when the specified pool for the subdomain does not exist
def get_ytimg_pool(subdomain)
if pool = YTIMG_POOLS[subdomain]?
return pool
else
LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"")
pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size)
YTIMG_POOLS[subdomain] = pool
return pool
end
end