From c24ed85110cfa006992ce16bd4432eb39c8db71b Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 16 Jun 2024 14:49:48 -0700 Subject: [PATCH 01/30] Fix named arg syntax when passing force_resolve --- src/invidious/routes/video_playback.cr | 8 ++++---- src/invidious/yt_backend/connection_pool.cr | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index ec18f3b8..254c0b46 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -42,7 +42,7 @@ module Invidious::Routes::VideoPlayback headers["Range"] = "bytes=#{range_for_head}" end - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) response = HTTP::Client::Response.new(500) error = "" 5.times do @@ -57,7 +57,7 @@ module Invidious::Routes::VideoPlayback if new_host != host host = new_host client.close - client = make_client(URI.parse(new_host), region, force_resolve = true) + client = make_client(URI.parse(new_host), region, force_resolve: true) end url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -71,7 +71,7 @@ module Invidious::Routes::VideoPlayback fvip = "3" host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) rescue ex error = ex.message end @@ -196,7 +196,7 @@ module Invidious::Routes::VideoPlayback break else client.close - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) end end diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index d3dbcc0e..bcf6a003 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -70,7 +70,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) end def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) - client = make_client(url, region, force_resolve) + client = make_client(url, region, force_resolve: force_resolve) begin yield client ensure From 1124dd645d0db872b01a0c476c205da057a8fd04 Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 22 May 2024 11:29:28 -0700 Subject: [PATCH 02/30] Use `make_client` instead of calling `HTTP::Client` Using `make_client` to create `HTTP::Client`, allows for a simple way to easily add logic to all `HTTP::Client` initialized within Invidious. --- src/invidious/routes/api/v1/search.cr | 4 +--- src/invidious/yt_backend/connection_pool.cr | 13 ++++--------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 2922b060..6785ef73 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -31,9 +31,7 @@ module Invidious::Routes::API::V1::Search query = env.params.query["q"]? || "" begin - client = HTTP::Client.new("suggestqueries-clients6.youtube.com") - client.before_request { |r| add_yt_headers(r) } - + client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers = true) url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt" response = client.get(url).body diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index ca612083..84d857ec 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -30,11 +30,8 @@ struct YoutubeConnectionPool response = yield conn rescue ex conn.close - conn = HTTP::Client.new(url) - - conn.family = CONFIG.force_resolve + conn = make_client(url) conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" response = yield conn ensure pool.release(conn) @@ -45,16 +42,14 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - conn = HTTP::Client.new(url) - conn.family = CONFIG.force_resolve + conn = make_client(url, force_solve = true) conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" conn end end end -def make_client(url : URI, region = nil, force_resolve : Bool = false) +def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_header : Bool = false) client = HTTP::Client.new(url) # Force the usage of a specific configured IP Family @@ -62,7 +57,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) client.family = CONFIG.force_resolve end - client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_header client.read_timeout = 10.seconds client.connect_timeout = 10.seconds From 3af668186997d21295247ed6e31c6fd4634fa511 Mon Sep 17 00:00:00 2001 From: syeopite <70992037+syeopite@users.noreply.github.com> Date: Fri, 24 May 2024 13:11:14 -0700 Subject: [PATCH 03/30] Fix typo in argument to `make_client` Co-authored-by: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> --- src/invidious/yt_backend/connection_pool.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 84d857ec..8563cc3e 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -42,7 +42,7 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - conn = make_client(url, force_solve = true) + conn = make_client(url, force_resolve = true) conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn end From ee89db49ba6242771921c9204d57f47f3edb8975 Mon Sep 17 00:00:00 2001 From: syeopite <70992037+syeopite@users.noreply.github.com> Date: Sun, 16 Jun 2024 15:18:21 +0000 Subject: [PATCH 04/30] Typo Co-authored-by: Samantaz Fox --- src/invidious/yt_backend/connection_pool.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 8563cc3e..f7227d67 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -42,7 +42,7 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - conn = make_client(url, force_resolve = true) + conn = make_client(url, force_resolve: true) conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn end From bd48af825c27f08987ee039381a702ca91e52cb8 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 16 Jun 2024 14:15:05 -0700 Subject: [PATCH 05/30] Search API: Fix named arg syntax to make_client --- src/invidious/routes/api/v1/search.cr | 2 +- src/invidious/yt_backend/connection_pool.cr | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 6785ef73..59a30745 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Search query = env.params.query["q"]? || "" begin - client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers = true) + client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true) url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt" response = client.get(url).body diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index f7227d67..0dc42261 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -49,7 +49,7 @@ struct YoutubeConnectionPool end end -def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_header : Bool = false) +def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false) client = HTTP::Client.new(url) # Force the usage of a specific configured IP Family @@ -57,7 +57,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, force_you client.family = CONFIG.force_resolve end - client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_header + client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_headers client.read_timeout = 10.seconds client.connect_timeout = 10.seconds From 7521902e88a4654378b5a0428f9c538d52dcb9db Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 24 Aug 2024 19:37:04 -0700 Subject: [PATCH 06/30] Ensure IP family is always used when force_resolve --- src/invidious/yt_backend/connection_pool.cr | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 0dc42261..eaa94158 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -31,7 +31,6 @@ struct YoutubeConnectionPool rescue ex conn.close conn = make_client(url) - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC response = yield conn ensure pool.release(conn) @@ -42,9 +41,7 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - conn = make_client(url, force_resolve: true) - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn + next make_client(url, force_resolve: true) end end end @@ -55,6 +52,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, force_you # Force the usage of a specific configured IP Family if force_resolve client.family = CONFIG.force_resolve + client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC end client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_headers From 46c58bd84cf2a867b897338bb2105648aed0118c Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 24 Aug 2024 19:38:02 -0700 Subject: [PATCH 07/30] Pool: Use force_resolve in fallback new client --- src/invidious/yt_backend/connection_pool.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index eaa94158..d474d760 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -30,7 +30,7 @@ struct YoutubeConnectionPool response = yield conn rescue ex conn.close - conn = make_client(url) + conn = make_client(url, force_resolve: true) response = yield conn ensure pool.release(conn) From 6e39b9b303930f931b5a5a60c75528c4d9db3587 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 24 Aug 2024 19:41:39 -0700 Subject: [PATCH 08/30] make_client: add YouTube headers on *.youtube.com --- src/invidious/yt_backend/connection_pool.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index d474d760..bff4df72 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -55,7 +55,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, force_you client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC end - client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" || force_youtube_headers + client.before_request { |r| add_yt_headers(r) } if url.host.try &.ends_with?("youtube.com") || force_youtube_headers client.read_timeout = 10.seconds client.connect_timeout = 10.seconds From f8ec3123286e63148123ccba781dcd699f705e1d Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 19 Sep 2024 21:24:20 -0300 Subject: [PATCH 09/30] Logger: Add color support for different log levels --- config/config.example.yml | 9 +++++++++ src/invidious.cr | 5 ++++- src/invidious/config.cr | 2 ++ src/invidious/helpers/logger.cr | 19 +++++++++++++++++-- 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 219aa03f..37b932ea 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -222,6 +222,15 @@ https_only: false ## #log_level: Info +## +## Enables colors in logs. Useful for debugging purposes +## This is overridden if "-k" or "--colorize" +## are passed on the command line. +## +## Accepted values: true, false +## Default: false +## +#colorize_logs: false # ----------------------------- # Features diff --git a/src/invidious.cr b/src/invidious.cr index 3804197e..d9a479d1 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -117,6 +117,9 @@ Kemal.config.extra_options do |parser| parser.on("-l LEVEL", "--log-level=LEVEL", "Log level, one of #{LogLevel.values} (default: #{CONFIG.log_level})") do |log_level| CONFIG.log_level = LogLevel.parse(log_level) end + parser.on("-k", "--colorize", "Colorize logs") do + CONFIG.colorize_logs = true + end parser.on("-v", "--version", "Print version") do puts SOFTWARE.to_pretty_json exit @@ -133,7 +136,7 @@ if CONFIG.output.upcase != "STDOUT" FileUtils.mkdir_p(File.dirname(CONFIG.output)) end OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a") -LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) +LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level, CONFIG.colorize_logs) # Check table integrity Invidious::Database.check_integrity(CONFIG) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index c4ddcdb3..d8543d35 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -68,6 +68,8 @@ class Config property output : String = "STDOUT" # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr property log_level : LogLevel = LogLevel::Info + # Enables colors in logs. Useful for debugging purposes + property colorize_logs : Bool = false # Database configuration with separate parameters (username, hostname, etc) property db : DBConfig? = nil diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index b443073e..36a3a7f9 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -1,3 +1,5 @@ +require "colorize" + enum LogLevel All = 0 Trace = 1 @@ -10,7 +12,7 @@ enum LogLevel end class Invidious::LogHandler < Kemal::BaseLogHandler - def initialize(@io : IO = STDOUT, @level = LogLevel::Debug) + def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, @color : Bool = true) end def call(context : HTTP::Server::Context) @@ -39,10 +41,23 @@ class Invidious::LogHandler < Kemal::BaseLogHandler @io.flush end + def color(level) + case level + when LogLevel::Trace then :cyan + when LogLevel::Debug then :green + when LogLevel::Info then :white + when LogLevel::Warn then :yellow + when LogLevel::Error then :red + when LogLevel::Fatal then :magenta + else :default + end + end + {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) if LogLevel::{{level.id.capitalize}} >= @level - puts("#{Time.utc} [{{level.id}}] #{message}") + puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})).toggle(@color)) + end end {% end %} From d77afdcf00f55a4455fb84dd90c4e5773167b759 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Fri, 20 Sep 2024 00:32:27 -0300 Subject: [PATCH 10/30] Logger: Make colorize_logs true by default --- config/config.example.yml | 6 ++++-- src/invidious/config.cr | 2 +- src/invidious/helpers/logger.cr | 5 ++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 37b932ea..fefc28be 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -226,11 +226,13 @@ https_only: false ## Enables colors in logs. Useful for debugging purposes ## This is overridden if "-k" or "--colorize" ## are passed on the command line. +## Colors are also disabled if the environment variable +## NO_COLOR is present and has any value ## ## Accepted values: true, false -## Default: false +## Default: true ## -#colorize_logs: false +#colorize_logs: true # ----------------------------- # Features diff --git a/src/invidious/config.cr b/src/invidious/config.cr index d8543d35..054f8db7 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -69,7 +69,7 @@ class Config # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr property log_level : LogLevel = LogLevel::Info # Enables colors in logs. Useful for debugging purposes - property colorize_logs : Bool = false + property colorize_logs : Bool = true # Database configuration with separate parameters (username, hostname, etc) property db : DBConfig? = nil diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index 36a3a7f9..3c425ff4 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -12,7 +12,7 @@ enum LogLevel end class Invidious::LogHandler < Kemal::BaseLogHandler - def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, @color : Bool = true) + def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, @use_color : Bool = true) end def call(context : HTTP::Server::Context) @@ -56,8 +56,7 @@ class Invidious::LogHandler < Kemal::BaseLogHandler {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) if LogLevel::{{level.id.capitalize}} >= @level - puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})).toggle(@color)) - + puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})).toggle(@use_color)) end end {% end %} From 17b525f2a66f6e832ccdc74522feebe68f73d9de Mon Sep 17 00:00:00 2001 From: Fijxu Date: Fri, 27 Sep 2024 18:08:21 -0300 Subject: [PATCH 11/30] Logger: colorize_logs false by default --- config/config.example.yml | 2 +- src/invidious/config.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index fefc28be..d79622ad 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -232,7 +232,7 @@ https_only: false ## Accepted values: true, false ## Default: true ## -#colorize_logs: true +#colorize_logs: false # ----------------------------- # Features diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 054f8db7..d8543d35 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -69,7 +69,7 @@ class Config # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr property log_level : LogLevel = LogLevel::Info # Enables colors in logs. Useful for debugging purposes - property colorize_logs : Bool = true + property colorize_logs : Bool = false # Database configuration with separate parameters (username, hostname, etc) property db : DBConfig? = nil From d2edd4b63fe690c248ff8709b39098fcdad0e109 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Tue, 8 Oct 2024 18:36:50 -0300 Subject: [PATCH 12/30] fixup! Logger: Add color support for different log levels --- config/config.example.yml | 2 +- src/invidious/helpers/logger.cr | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index d79622ad..f746d1f7 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -224,7 +224,7 @@ https_only: false ## ## Enables colors in logs. Useful for debugging purposes -## This is overridden if "-k" or "--colorize" +## This is overridden if "-k" or "--colorize" ## are passed on the command line. ## Colors are also disabled if the environment variable ## NO_COLOR is present and has any value diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index 3c425ff4..03349595 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -12,7 +12,9 @@ enum LogLevel end class Invidious::LogHandler < Kemal::BaseLogHandler - def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, @use_color : Bool = true) + def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true) + Colorize.enabled = use_color + Colorize.on_tty_only! end def call(context : HTTP::Server::Context) @@ -56,7 +58,7 @@ class Invidious::LogHandler < Kemal::BaseLogHandler {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) if LogLevel::{{level.id.capitalize}} >= @level - puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})).toggle(@use_color)) + puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}}))) end end {% end %} From ee728092823d8e82f71f35c31da8a27efec0f1b5 Mon Sep 17 00:00:00 2001 From: Brahim Hadriche Date: Sat, 26 Oct 2024 12:40:31 -0400 Subject: [PATCH 13/30] [Alternative] Fix for channel live videos --- src/invidious/channels/videos.cr | 66 +++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 6cc30142..e29d80ed 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -23,29 +23,57 @@ def produce_channel_content_continuation(ucid, content_type, page = 1, auto_gene else 15 # Fallback to "videos" end - sort_by_numerical = - case sort_by - when "newest" then 1_i64 - when "popular" then 2_i64 - when "oldest" then 4_i64 - else 1_i64 # Fallback to "newest" - end + if content_type == "livestreams" + sort_by_numerical = + case sort_by + when "newest" then 12_i64 + when "popular" then 14_i64 + when "oldest" then 13_i64 + else 12_i64 # Fallback to "newest" + end + else + sort_by_numerical = + case sort_by + when "newest" then 1_i64 + when "popular" then 2_i64 + when "oldest" then 4_i64 + else 1_i64 # Fallback to "newest" + end + end - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "#{content_type_numerical}:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, + if content_type == "livestreams" + object_inner_1 = { + "110:embedded" => { + "3:embedded" => { + "#{content_type_numerical}:embedded" => { + "1:embedded" => { + "1:string" => object_inner_2_encoded, + }, + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "5:varint" => sort_by_numerical, }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "3:varint" => sort_by_numerical, }, }, - }, - } + } + else + object_inner_1 = { + "110:embedded" => { + "3:embedded" => { + "#{content_type_numerical}:embedded" => { + "1:embedded" => { + "1:string" => object_inner_2_encoded, + }, + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "3:varint" => sort_by_numerical, + }, + }, + }, + } + end object_inner_1_encoded = object_inner_1 .try { |i| Protodec::Any.cast_json(i) } From 711d52d47fcff4cc376551a81fba47dcaeb23e0c Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 29 Oct 2024 17:26:24 +0100 Subject: [PATCH 14/30] Shards: Update database dependencies --- shard.lock | 6 +++--- shard.yml | 4 ++-- src/invidious/yt_backend/connection_pool.cr | 9 ++++++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/shard.lock b/shard.lock index 397bd8bc..1f609631 100644 --- a/shard.lock +++ b/shard.lock @@ -14,7 +14,7 @@ shards: db: git: https://github.com/crystal-lang/crystal-db.git - version: 0.10.1 + version: 0.13.1 exception_page: git: https://github.com/crystal-loot/exception_page.git @@ -30,7 +30,7 @@ shards: pg: git: https://github.com/will/crystal-pg.git - version: 0.24.0 + version: 0.28.0 protodec: git: https://github.com/iv-org/protodec.git @@ -46,5 +46,5 @@ shards: sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git - version: 0.18.0 + version: 0.21.0 diff --git a/shard.yml b/shard.yml index 367f7c73..e0e34c0c 100644 --- a/shard.yml +++ b/shard.yml @@ -12,10 +12,10 @@ targets: dependencies: pg: github: will/crystal-pg - version: ~> 0.24.0 + version: ~> 0.28.0 sqlite3: github: crystal-lang/crystal-sqlite3 - version: ~> 0.18.0 + version: ~> 0.21.0 kemal: github: kemalcr/kemal version: ~> 1.1.2 diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index ca612083..14cc2d47 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -44,7 +44,14 @@ struct YoutubeConnectionPool end private def build_pool - DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do + options = DB::Pool::Options.new( + initial_pool_size: 0, + max_pool_size: capacity, + max_idle_pool_size: capacity, + checkout_timeout: timeout + ) + + DB::Pool(HTTP::Client).new(options) do conn = HTTP::Client.new(url) conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC From c243d08afb8509f7a98cd7aa1b77d4f409a7a823 Mon Sep 17 00:00:00 2001 From: Brahim Hadriche Date: Wed, 30 Oct 2024 13:38:13 -0400 Subject: [PATCH 15/30] refactor --- src/invidious/channels/videos.cr | 49 +++++++++++++------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index e29d80ed..bcdc8d8f 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -23,6 +23,13 @@ def produce_channel_content_continuation(ucid, content_type, page = 1, auto_gene else 15 # Fallback to "videos" end + sort_type_numerical = + case content_type + when "videos" then 3 + when "livestreams" then 5 + else 3 # Fallback to "videos" + end + if content_type == "livestreams" sort_by_numerical = case sort_by @@ -41,39 +48,21 @@ def produce_channel_content_continuation(ucid, content_type, page = 1, auto_gene end end - if content_type == "livestreams" - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "#{content_type_numerical}:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, - }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "5:varint" => sort_by_numerical, + object_inner_1 = { + "110:embedded" => { + "3:embedded" => { + "#{content_type_numerical}:embedded" => { + "1:embedded" => { + "1:string" => object_inner_2_encoded, }, + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "#{sort_type_numerical}:varint" => sort_by_numerical, }, }, - } - else - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "#{content_type_numerical}:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, - }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "3:varint" => sort_by_numerical, - }, - }, - }, - } - end + }, + } object_inner_1_encoded = object_inner_1 .try { |i| Protodec::Any.cast_json(i) } From 75c5881c553b8225f389db11733639ae62885c2f Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 31 Oct 2024 13:31:59 +0100 Subject: [PATCH 16/30] Locales: Add Bulgarian, Welsh and Lombard to the list --- src/invidious/helpers/i18n.cr | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 23a1aafc..1ba3ea61 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -1,8 +1,22 @@ +# Languages requiring a better level of translation (at least 20%) +# to be added to the list below: +# +# "af" => "", # Afrikaans +# "az" => "", # Azerbaijani +# "be" => "", # Belarusian +# "bn_BD" => "", # Bengali (Bangladesh) +# "ia" => "", # Interlingua +# "or" => "", # Odia +# "tk" => "", # Turkmen +# "tok => "", # Toki Pona +# LOCALES_LIST = { "ar" => "العربية", # Arabic + "bg" => "български", # Bulgarian "bn" => "বাংলা", # Bengali "ca" => "Català", # Catalan "cs" => "Čeština", # Czech + "cy" => "Cymraeg", # Welsh "da" => "Dansk", # Danish "de" => "Deutsch", # German "el" => "Ελληνικά", # Greek @@ -23,6 +37,7 @@ LOCALES_LIST = { "it" => "Italiano", # Italian "ja" => "日本語", # Japanese "ko" => "한국어", # Korean + "lmo" => "Lombard", # Lombard "lt" => "Lietuvių", # Lithuanian "nb-NO" => "Norsk bokmål", # Norwegian Bokmål "nl" => "Nederlands", # Dutch From ac6e796c732bb4be5a0fe6be9ba53ad49c49bd51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:04:43 +0100 Subject: [PATCH 17/30] checking the status code returned by youtube (#5052) * checking the status code returned by youtube * add documentation link * Update src/invidious/yt_backend/youtube_api.cr Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com> --------- Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com> --- src/invidious/yt_backend/youtube_api.cr | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index baa3cd92..e0a3181f 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -638,6 +638,11 @@ module YoutubeAPI # Send the POST request body = YT_POOL.client() do |client| client.post(url, headers: headers, body: data.to_json) do |response| + if response.status_code != 200 + raise InfoException.new("Error: non 200 status code. Youtube API returned \ + status code #{response.status_code}. See \ + https://docs.invidious.io/youtube-errors-explained/ for troubleshooting.") + end self._decompress(response.body_io, response.headers["Content-Encoding"]?) end end From 792d0d5f6df912039a58768e6ff503ae00abe7c0 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 7 Nov 2024 13:14:36 +0000 Subject: [PATCH 18/30] CI: Check Crystal lint only on latest version (#5042) * CI: Check Crystal lint only on latest version * Apply suggestion from code review Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com> --------- Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com> --- .github/workflows/ci.yml | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 411ec769..dd472d1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: - name: Install required APT packages run: | - sudo apt install -y libsqlite3-dev + sudo apt install -y libsqlite3-dev shell: bash - name: Install Crystal @@ -65,7 +65,9 @@ jobs: - name: Cache Shards uses: actions/cache@v3 with: - path: ./lib + path: | + ./lib + ./bin key: shards-${{ hashFiles('shard.lock') }} - name: Install Shards @@ -77,14 +79,6 @@ jobs: - name: Run tests run: crystal spec - - name: Run lint - run: | - if ! crystal tool format --check; then - crystal tool format - git diff - exit 1 - fi - - name: Build run: crystal build --warnings all --error-on-warnings --error-trace src/invidious.cr @@ -130,8 +124,12 @@ jobs: - name: Test Docker run: while curl -Isf http://localhost:3000; do sleep 1; done - ameba_lint: + lint: + runs-on: ubuntu-latest + + continue-on-error: true + steps: - uses: actions/checkout@v4 with: @@ -151,7 +149,18 @@ jobs: key: shards-${{ hashFiles('shard.lock') }} - name: Install Shards - run: shards install + run: | + if ! shards check; then + shards install + fi + + - name: Check Crystal formatter compliance + run: | + if ! crystal tool format --check; then + crystal tool format + git diff + exit 1 + fi - name: Run Ameba linter run: bin/ameba From cbc546f0320e4833927a654c26d384bb2e8a9f93 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 7 Nov 2024 22:54:21 +0100 Subject: [PATCH 19/30] Channels: Add function to generate the new ctoken objects --- src/invidious/channels/videos.cr | 207 +++++++++++++++---------------- 1 file changed, 103 insertions(+), 104 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index bcdc8d8f..7b3e3cfa 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,95 +1,3 @@ -def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - object_inner_2 = { - "2:0:embedded" => { - "1:0:varint" => 0_i64, - }, - "5:varint" => 50_i64, - "6:varint" => 1_i64, - "7:varint" => (page * 30).to_i64, - "9:varint" => 1_i64, - "10:varint" => 0_i64, - } - - object_inner_2_encoded = object_inner_2 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - content_type_numerical = - case content_type - when "videos" then 15 - when "livestreams" then 14 - else 15 # Fallback to "videos" - end - - sort_type_numerical = - case content_type - when "videos" then 3 - when "livestreams" then 5 - else 3 # Fallback to "videos" - end - - if content_type == "livestreams" - sort_by_numerical = - case sort_by - when "newest" then 12_i64 - when "popular" then 14_i64 - when "oldest" then 13_i64 - else 12_i64 # Fallback to "newest" - end - else - sort_by_numerical = - case sort_by - when "newest" then 1_i64 - when "popular" then 2_i64 - when "oldest" then 4_i64 - else 1_i64 # Fallback to "newest" - end - end - - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "#{content_type_numerical}:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, - }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "#{sort_type_numerical}:varint" => sort_by_numerical, - }, - }, - }, - } - - object_inner_1_encoded = object_inner_1 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:string" => object_inner_1_encoded, - "35:string" => "browse-feed#{ucid}videos102", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def make_initial_content_ctoken(ucid, content_type, sort_by) : String - return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by) -end - module Invidious::Channel::Tabs extend self @@ -118,7 +26,7 @@ module Invidious::Channel::Tabs end def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by) + continuation ||= make_videos_ctoken(ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, author, ucid) @@ -147,14 +55,10 @@ module Invidious::Channel::Tabs # Shorts # ------------------- - def get_shorts(channel : AboutChannel, continuation : String? = nil) - if continuation.nil? - # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts" - # TODO: try to extract the continuation tokens that allows other sorting options - initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation: continuation) - end + def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_shorts_ctoken(channel.ucid, sort_by) + initial_data = YoutubeAPI.browse(continuation: continuation) + return extract_items(initial_data, channel.author, channel.ucid) end @@ -162,9 +66,8 @@ module Invidious::Channel::Tabs # Livestreams # ------------------- - def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by) - + def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_livestreams_ctoken(channel.ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, channel.author, channel.ucid) @@ -188,4 +91,100 @@ module Invidious::Channel::Tabs return items, next_continuation end + + # ------------------- + # C-tokens + # ------------------- + + private def sort_options_videos_short(sort_by : String) + case sort_by + when "newest" then return 4_i64 + when "popular" then return 2_i64 + when "oldest" then return 5_i64 + else return 4_i64 # Fallback to "newest" + end + end + + # Generate the initial "continuation token" to get the first page of the + # "videos" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_videos_ctoken(ucid : String, sort_by = "newest") + object = { + "15:embedded" => { + "2:string" => "\n$00000000-0000-0000-0000-000000000000", + "4:varint" => sort_options_videos_short(sort_by), + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # Generate the initial "continuation token" to get the first page of the + # "shorts" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_shorts_ctoken(ucid : String, sort_by = "newest") + object = { + "10:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "4:varint" => sort_options_videos_short(sort_by), + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # Generate the initial "continuation token" to get the first page of the + # "livestreams" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_livestreams_ctoken(ucid : String, sort_by = "newest") + sort_by_numerical = + case sort_by + when "newest" then 12_i64 + when "popular" then 14_i64 + when "oldest" then 13_i64 + else 12_i64 # Fallback to "newest" + end + + object = { + "14:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "5:varint" => sort_by_numerical, + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # The protobuf structure common between videos/shorts/livestreams + private def channel_ctoken_wrap(ucid : String, object) + object_inner = { + "110:embedded" => { + "3:embedded" => object, + }, + } + + object_inner_encoded = object_inner + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => object_inner_encoded, + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation + end end From 82248fad024de5289011e2ae26d5c390d5084827 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 7 Nov 2024 23:00:18 +0100 Subject: [PATCH 20/30] Channels: Add sort options to shorts --- src/invidious/routes/channels.cr | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 952098e0..d4e9fa68 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -82,13 +82,12 @@ module Invidious::Routes::Channels end next_continuation = nil else - # TODO: support sort option for shorts - sort_by = "" - sort_options = [] of String + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} # Fetch items and continuation token items, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) end From 3196182d4d303520d8d49b915a70bdc0c8fb12cd Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 7 Nov 2024 20:41:04 -0800 Subject: [PATCH 21/30] Prevent PRs from being considered stale --- .github/workflows/stale.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 16d3269b..35740d60 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -14,13 +14,10 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 365 - days-before-pr-stale: 90 + days-before-pr-stale: -1 days-before-close: 30 - exempt-pr-labels: blocked,exempt-stale stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.' - stale-pr-message: 'This pull request has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely abandoned or outdated. If you think this pull request is still relevant and applicable, you just have to post a comment and it will be unmarked.' stale-issue-label: "stale" - stale-pr-label: "stale" ascending: true # Never mark feature requests/enhancements as stale exempt-issue-labels: "feature-request,enhancement,exempt-stale" From 78f18b257c7099d1d22b7566b62d5c2d596fa27f Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 7 Nov 2024 20:42:19 -0800 Subject: [PATCH 22/30] Double stale timer for issues Days before staling is increased to 730 days Days before closing is increased to 60 days --- .github/workflows/stale.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 35740d60..1294804a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,9 +13,9 @@ jobs: - uses: actions/stale@v8 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - days-before-stale: 365 + days-before-stale: 730 days-before-pr-stale: -1 - days-before-close: 30 + days-before-close: 60 stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.' stale-issue-label: "stale" ascending: true From ce910b5269c57658ef44d46e4024dc623f5ed46d Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 7 Nov 2024 20:45:23 -0800 Subject: [PATCH 23/30] Prevent discussion issues from being staled --- .github/workflows/stale.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1294804a..498a2c1b 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -19,5 +19,5 @@ jobs: stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.' stale-issue-label: "stale" ascending: true - # Never mark feature requests/enhancements as stale - exempt-issue-labels: "feature-request,enhancement,exempt-stale" + # Exempt the following types of issues from being staled + exempt-issue-labels: "feature-request,enhancement,discussion,exempt-stale" From 1a5047aad94454fd8a8d9623e17ee3782c68c3d0 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 8 Nov 2024 12:33:14 +0100 Subject: [PATCH 24/30] Extractors: Add support for lockupViewModel The 'lockupViewModel' structure is used in the channel "podcasts" tab --- src/invidious/yt_backend/extractors.cr | 76 +++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 4074de86..cb8331a5 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -467,9 +467,9 @@ private module Parsers # Parses an InnerTube richItemRenderer into a SearchVideo. # Returns nil when the given object isn't a RichItemRenderer # - # A richItemRenderer seems to be a simple wrapper for a videoRenderer, used - # by the result page for hashtags and for the podcast tab on channels. - # It is located inside a continuationItems container for hashtags. + # A richItemRenderer seems to be a simple wrapper for a various other types, + # used on the hashtags result page and the channel podcast tab. It is located + # itself inside a richGridRenderer container. # module RichItemRendererParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) @@ -482,6 +482,7 @@ private module Parsers child = VideoRendererParser.process(item_contents, author_fallback) child ||= ReelItemRendererParser.process(item_contents, author_fallback) child ||= PlaylistRendererParser.process(item_contents, author_fallback) + child ||= LockupViewModelParser.process(item_contents, author_fallback) return child end @@ -582,6 +583,75 @@ private module Parsers end end + # Parses an InnerTube lockupViewModel into a SearchPlaylist. + # Returns nil when the given object is not a lockupViewModel. + # + # This structure is present since November 2024 on the "podcasts" tab of the + # channel page. It is usually (always?) encapsulated in a richItemRenderer. + # + module LockupViewModelParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["lockupViewModel"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + playlist_id = item_contents["contentId"].as_s + + thumbnail_view_model = item_contents.dig( + "contentImage", "collectionThumbnailViewModel", + "primaryThumbnail", "thumbnailViewModel" + ) + + thumbnail = thumbnail_view_model.dig("image", "sources", 1, "url").as_s + + # This complicated sequences tries to extract the following data structure: + # "overlays": [{ + # "thumbnailOverlayBadgeViewModel": { + # "thumbnailBadges": [{ + # "thumbnailBadgeViewModel": { + # "text": "430 episodes", + # "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT" + # } + # }] + # } + # }] + video_count = thumbnail_view_model.dig("overlays").as_a + .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a) + .flatten + .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try &.as_s.ends_with?("episodes")) + .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false) + + metadata = item_contents.dig("metadata", "lockupMetadataViewModel") + title = metadata.dig("title", "content").as_s + + # TODO: Retrieve "updated" info from metadata parts + # rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a + # parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s) + # One of these parts should contain a string like: "Updated 2 days ago" + + # TODO: Maybe add a button to access the first video of the playlist? + # item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint") + # Available fields: "videoId", "playlistId", "params" + + return SearchPlaylist.new({ + title: title, + id: playlist_id, + author: author_fallback.name, + ucid: author_fallback.id, + video_count: video_count || -1, + videos: [] of SearchPlaylistVideo, + thumbnail: thumbnail, + author_verified: false, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + # Parses an InnerTube continuationItemRenderer into a Continuation. # Returns nil when the given object isn't a continuationItemRenderer. # From afc5b27d83d8b2b287842ed1ec43185135441d37 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 8 Nov 2024 13:32:44 +0100 Subject: [PATCH 25/30] Extractors: Add support for shortsLockupViewModel The 'shortsLockupViewModel' structure is used in the channel "shorts" tab --- src/invidious/yt_backend/extractors.cr | 58 ++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index cb8331a5..4416ef30 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -483,6 +483,7 @@ private module Parsers child ||= ReelItemRendererParser.process(item_contents, author_fallback) child ||= PlaylistRendererParser.process(item_contents, author_fallback) child ||= LockupViewModelParser.process(item_contents, author_fallback) + child ||= ShortsLockupViewModelParser.process(item_contents, author_fallback) return child end @@ -497,6 +498,9 @@ private module Parsers # reelItemRenderer items are used in the new (2022) channel layout, # in the "shorts" tab. # + # NOTE: As of 10/2024, it might have been fully replaced by shortsLockupViewModel + # TODO: Confirm that hypothesis + # module ReelItemRendererParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["reelItemRenderer"]? @@ -652,6 +656,60 @@ private module Parsers end end + # Parses an InnerTube shortsLockupViewModel into a SearchVideo. + # Returns nil when the given object is not a shortsLockupViewModel. + # + # This structure is present since around October 2024 on the "shorts" tab of + # the channel page and likely replaces the reelItemRenderer structure. It is + # usually (always?) encapsulated in a richItemRenderer. + # + module ShortsLockupViewModelParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["shortsLockupViewModel"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + # TODO: Maybe add support for "oardefault.jpg" thumbnails? + # thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s + # Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?... + + video_id = item_contents.dig( + "onTap", "innertubeCommand", "reelWatchEndpoint", "videoId" + ).as_s + + title = item_contents.dig("overlayMetadata", "primaryText", "content").as_s + + view_count = short_text_to_number( + item_contents.dig("overlayMetadata", "secondaryText", "content").as_s + ) + + # Approximate to one minute, as "shorts" generally don't exceed that. + # NOTE: The actual duration is not provided by Youtube anymore. + # TODO: Maybe use -1 as an error value and handle that on the frontend? + duration = 60_i32 + + SearchVideo.new({ + title: title, + id: video_id, + author: author_fallback.name, + ucid: author_fallback.id, + published: Time.unix(0), + views: view_count, + description_html: "", + length_seconds: duration, + premiere_timestamp: Time.unix(0), + author_verified: false, + badges: VideoBadges::None, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + # Parses an InnerTube continuationItemRenderer into a Continuation. # Returns nil when the given object isn't a continuationItemRenderer. # From d27a5e7fae4a826b66950422ff8dfec4123dabf1 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 8 Nov 2024 13:33:46 +0100 Subject: [PATCH 26/30] Channels: Rename ctoken generator functions as requested --- src/invidious/channels/videos.cr | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 7b3e3cfa..9572adf3 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -26,7 +26,7 @@ module Invidious::Channel::Tabs end def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_videos_ctoken(ucid, sort_by) + continuation ||= make_initial_videos_ctoken(ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, author, ucid) @@ -56,7 +56,7 @@ module Invidious::Channel::Tabs # ------------------- def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_shorts_ctoken(channel.ucid, sort_by) + continuation ||= make_initial_shorts_ctoken(channel.ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, channel.author, channel.ucid) @@ -67,7 +67,7 @@ module Invidious::Channel::Tabs # ------------------- def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_livestreams_ctoken(channel.ucid, sort_by) + continuation ||= make_initial_livestreams_ctoken(channel.ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, channel.author, channel.ucid) @@ -108,7 +108,7 @@ module Invidious::Channel::Tabs # Generate the initial "continuation token" to get the first page of the # "videos" tab. The following page requires the ctoken provided in that # first page, and so on. - private def make_videos_ctoken(ucid : String, sort_by = "newest") + private def make_initial_videos_ctoken(ucid : String, sort_by = "newest") object = { "15:embedded" => { "2:string" => "\n$00000000-0000-0000-0000-000000000000", @@ -122,7 +122,7 @@ module Invidious::Channel::Tabs # Generate the initial "continuation token" to get the first page of the # "shorts" tab. The following page requires the ctoken provided in that # first page, and so on. - private def make_shorts_ctoken(ucid : String, sort_by = "newest") + private def make_initial_shorts_ctoken(ucid : String, sort_by = "newest") object = { "10:embedded" => { "2:embedded" => { @@ -138,7 +138,7 @@ module Invidious::Channel::Tabs # Generate the initial "continuation token" to get the first page of the # "livestreams" tab. The following page requires the ctoken provided in that # first page, and so on. - private def make_livestreams_ctoken(ucid : String, sort_by = "newest") + private def make_initial_livestreams_ctoken(ucid : String, sort_by = "newest") sort_by_numerical = case sort_by when "newest" then 12_i64 From 301aeffa780fca321793f8c2ef46844d613ce5c3 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 8 Nov 2024 13:54:05 +0100 Subject: [PATCH 27/30] Channels: Multiple small fixes Fix the "newest" link not being bold when 'sort_by' uses the default value Show 60 videos per page, rather than 30 --- src/invidious/routes/channels.cr | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index d4e9fa68..7d634cbb 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -20,10 +20,11 @@ module Invidious::Routes::Channels sort_by = env.params.query["sort_by"]?.try &.downcase if channel.auto_generated + sort_by ||= "last" sort_options = {"last", "oldest", "newest"} items, next_continuation = fetch_channel_playlists( - channel.ucid, channel.author, continuation, (sort_by || "last") + channel.ucid, channel.author, continuation, sort_by ) items.uniq! do |item| @@ -49,9 +50,11 @@ module Invidious::Routes::Channels end next_continuation = nil else + sort_by ||= "newest" sort_options = {"newest", "oldest", "popular"} - items, next_continuation = Channel::Tabs.get_videos( - channel, continuation: continuation, sort_by: (sort_by || "newest") + + items, next_continuation = Channel::Tabs.get_60_videos( + channel, continuation: continuation, sort_by: sort_by ) end end From 6dd662a5b84b3deb9e19e365f8b480357f63a2e9 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 8 Nov 2024 17:25:23 +0100 Subject: [PATCH 28/30] Channels: lockupViewModel is also used in the "playlists" tab --- src/invidious/yt_backend/extractors.cr | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 4416ef30..2631b62a 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -21,6 +21,7 @@ private ITEM_PARSERS = { Parsers::ItemSectionRendererParser, Parsers::ContinuationItemRendererParser, Parsers::HashtagRendererParser, + Parsers::LockupViewModelParser, } private alias InitialData = Hash(String, JSON::Any) @@ -590,8 +591,9 @@ private module Parsers # Parses an InnerTube lockupViewModel into a SearchPlaylist. # Returns nil when the given object is not a lockupViewModel. # - # This structure is present since November 2024 on the "podcasts" tab of the - # channel page. It is usually (always?) encapsulated in a richItemRenderer. + # This structure is present since November 2024 on the "podcasts" and + # "playlists" tabs of the channel page. It is usually encapsulated in either + # a richItemRenderer or a richGridRenderer. # module LockupViewModelParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) @@ -608,7 +610,7 @@ private module Parsers "primaryThumbnail", "thumbnailViewModel" ) - thumbnail = thumbnail_view_model.dig("image", "sources", 1, "url").as_s + thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s # This complicated sequences tries to extract the following data structure: # "overlays": [{ @@ -621,10 +623,15 @@ private module Parsers # }] # } # }] + # + # NOTE: this simplistic `.to_i` conversion might not work on larger + # playlists and hasn't been tested. video_count = thumbnail_view_model.dig("overlays").as_a .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a) .flatten - .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try &.as_s.ends_with?("episodes")) + .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try { |node| + {"episodes", "videos"}.any? { |str| node.as_s.ends_with?(str) } + }) .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false) metadata = item_contents.dig("metadata", "lockupMetadataViewModel") From 2a19dbb1fee20e5438751c3bb387f8757f4c2238 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 8 Nov 2024 18:28:55 +0100 Subject: [PATCH 29/30] Channels: Use the same structure as in the other ctoken functions Change explanation, courtesy of iBicha: The \n is basically a decimal 10, which is 1010 binary. That is a field number 1, and a wire type 2 (length-delimited). Then the $ is a decimal 36, which is exactly the length of 00000000-0000-0000-0000-000000000000. So both objects end up being encoded into the same data. --- src/invidious/channels/videos.cr | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 9572adf3..96400f47 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -111,7 +111,9 @@ module Invidious::Channel::Tabs private def make_initial_videos_ctoken(ucid : String, sort_by = "newest") object = { "15:embedded" => { - "2:string" => "\n$00000000-0000-0000-0000-000000000000", + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, "4:varint" => sort_options_videos_short(sort_by), }, } From b173d4acf21563d47d26718eca7932878fb424e6 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 8 Nov 2024 23:45:15 +0100 Subject: [PATCH 30/30] Update CHANGELOG.md --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9892e17..061f977c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ ### Full list of pull requests merged since the last release (newest first) +* Stale bot updates ([#5060], thanks @syeopite) +* Channels: Fix "Youtube API returned error 400" ([#5059], by @SamantazFox) +* Channels: Fix for live videos ([#5027], thanks @iBicha) +* Locales: Add Bulgarian, Welsh and Lombard to the list ([#5046], by @SamantazFox) +* Shards: Update database dependencies ([#5034], by @SamantazFox) +* Logger: Add color support for different log levels ([#4931], thanks @Fijxu) +* Fix named arg syntax when passing force_resolve ([#4754], thanks @syeopite) +* Use make_client instead of calling HTTP::Client ([#4709], thanks @syeopite) * Add "Filipino (auto-generated)" to the list of caption languages ([#4995], by @SamantazFox) * Makefile: Add MT option to enable the 'preview_mt' flag ([#4993], by @SamantazFox) * SigHelper: Reconnect to signature helper ([#4991], thanks @Fijxu) @@ -31,7 +39,9 @@ [#4270]: https://github.com/iv-org/invidious/pull/4270 [#4326]: https://github.com/iv-org/invidious/pull/4326 [#4652]: https://github.com/iv-org/invidious/pull/4652 +[#4709]: https://github.com/iv-org/invidious/pull/4709 [#4750]: https://github.com/iv-org/invidious/pull/4750 +[#4754]: https://github.com/iv-org/invidious/pull/4754 [#4850]: https://github.com/iv-org/invidious/pull/4850 [#4862]: https://github.com/iv-org/invidious/pull/4862 [#4863]: https://github.com/iv-org/invidious/pull/4863 @@ -41,10 +51,16 @@ [#4923]: https://github.com/iv-org/invidious/pull/4923 [#4928]: https://github.com/iv-org/invidious/pull/4928 [#4930]: https://github.com/iv-org/invidious/pull/4930 +[#4931]: https://github.com/iv-org/invidious/pull/4931 [#4942]: https://github.com/iv-org/invidious/pull/4942 [#4991]: https://github.com/iv-org/invidious/pull/4991 [#4993]: https://github.com/iv-org/invidious/pull/4993 [#4995]: https://github.com/iv-org/invidious/pull/4995 +[#5027]: https://github.com/iv-org/invidious/pull/5027 +[#5034]: https://github.com/iv-org/invidious/pull/5034 +[#5046]: https://github.com/iv-org/invidious/pull/5046 +[#5059]: https://github.com/iv-org/invidious/pull/5059 +[#5060]: https://github.com/iv-org/invidious/pull/5060 ## v2.20240825.2 (2024-08-26)