diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 411ec769..dd472d1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: - name: Install required APT packages run: | - sudo apt install -y libsqlite3-dev + sudo apt install -y libsqlite3-dev shell: bash - name: Install Crystal @@ -65,7 +65,9 @@ jobs: - name: Cache Shards uses: actions/cache@v3 with: - path: ./lib + path: | + ./lib + ./bin key: shards-${{ hashFiles('shard.lock') }} - name: Install Shards @@ -77,14 +79,6 @@ jobs: - name: Run tests run: crystal spec - - name: Run lint - run: | - if ! crystal tool format --check; then - crystal tool format - git diff - exit 1 - fi - - name: Build run: crystal build --warnings all --error-on-warnings --error-trace src/invidious.cr @@ -130,8 +124,12 @@ jobs: - name: Test Docker run: while curl -Isf http://localhost:3000; do sleep 1; done - ameba_lint: + lint: + runs-on: ubuntu-latest + + continue-on-error: true + steps: - uses: actions/checkout@v4 with: @@ -151,7 +149,18 @@ jobs: key: shards-${{ hashFiles('shard.lock') }} - name: Install Shards - run: shards install + run: | + if ! shards check; then + shards install + fi + + - name: Check Crystal formatter compliance + run: | + if ! crystal tool format --check; then + crystal tool format + git diff + exit 1 + fi - name: Run Ameba linter run: bin/ameba diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 16d3269b..498a2c1b 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,14 +13,11 @@ jobs: - uses: actions/stale@v8 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - days-before-stale: 365 - days-before-pr-stale: 90 - days-before-close: 30 - exempt-pr-labels: blocked,exempt-stale + days-before-stale: 730 + days-before-pr-stale: -1 + days-before-close: 60 stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.' - stale-pr-message: 'This pull request has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely abandoned or outdated. If you think this pull request is still relevant and applicable, you just have to post a comment and it will be unmarked.' stale-issue-label: "stale" - stale-pr-label: "stale" ascending: true - # Never mark feature requests/enhancements as stale - exempt-issue-labels: "feature-request,enhancement,exempt-stale" + # Exempt the following types of issues from being staled + exempt-issue-labels: "feature-request,enhancement,discussion,exempt-stale" diff --git a/CHANGELOG.md b/CHANGELOG.md index f9892e17..061f977c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ ### Full list of pull requests merged since the last release (newest first) +* Stale bot updates ([#5060], thanks @syeopite) +* Channels: Fix "Youtube API returned error 400" ([#5059], by @SamantazFox) +* Channels: Fix for live videos ([#5027], thanks @iBicha) +* Locales: Add Bulgarian, Welsh and Lombard to the list ([#5046], by @SamantazFox) +* Shards: Update database dependencies ([#5034], by @SamantazFox) +* Logger: Add color support for different log levels ([#4931], thanks @Fijxu) +* Fix named arg syntax when passing force_resolve ([#4754], thanks @syeopite) +* Use make_client instead of calling HTTP::Client ([#4709], thanks @syeopite) * Add "Filipino (auto-generated)" to the list of caption languages ([#4995], by @SamantazFox) * Makefile: Add MT option to enable the 'preview_mt' flag ([#4993], by @SamantazFox) * SigHelper: Reconnect to signature helper ([#4991], thanks @Fijxu) @@ -31,7 +39,9 @@ [#4270]: https://github.com/iv-org/invidious/pull/4270 [#4326]: https://github.com/iv-org/invidious/pull/4326 [#4652]: https://github.com/iv-org/invidious/pull/4652 +[#4709]: https://github.com/iv-org/invidious/pull/4709 [#4750]: https://github.com/iv-org/invidious/pull/4750 +[#4754]: https://github.com/iv-org/invidious/pull/4754 [#4850]: https://github.com/iv-org/invidious/pull/4850 [#4862]: https://github.com/iv-org/invidious/pull/4862 [#4863]: https://github.com/iv-org/invidious/pull/4863 @@ -41,10 +51,16 @@ [#4923]: https://github.com/iv-org/invidious/pull/4923 [#4928]: https://github.com/iv-org/invidious/pull/4928 [#4930]: https://github.com/iv-org/invidious/pull/4930 +[#4931]: https://github.com/iv-org/invidious/pull/4931 [#4942]: https://github.com/iv-org/invidious/pull/4942 [#4991]: https://github.com/iv-org/invidious/pull/4991 [#4993]: https://github.com/iv-org/invidious/pull/4993 [#4995]: https://github.com/iv-org/invidious/pull/4995 +[#5027]: https://github.com/iv-org/invidious/pull/5027 +[#5034]: https://github.com/iv-org/invidious/pull/5034 +[#5046]: https://github.com/iv-org/invidious/pull/5046 +[#5059]: https://github.com/iv-org/invidious/pull/5059 +[#5060]: https://github.com/iv-org/invidious/pull/5060 ## v2.20240825.2 (2024-08-26) diff --git a/config/config.example.yml b/config/config.example.yml index eeb4501e..f5beaf81 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -235,11 +235,13 @@ http_proxy: ## ## Enables colors in logs. Useful for debugging purposes -## This is overridden if "-k" or "--colorize" +## This is overridden if "-k" or "--colorize" ## are passed on the command line. +## Colors are also disabled if the environment variable +## NO_COLOR is present and has any value ## ## Accepted values: true, false -## Default: false +## Default: true ## #colorize_logs: false diff --git a/shard.lock b/shard.lock index 8c4be868..1c396a7f 100644 --- a/shard.lock +++ b/shard.lock @@ -14,7 +14,7 @@ shards: db: git: https://github.com/crystal-lang/crystal-db.git - version: 0.10.1 + version: 0.13.1 exception_page: git: https://github.com/crystal-loot/exception_page.git @@ -37,7 +37,7 @@ shards: pg: git: https://github.com/will/crystal-pg.git - version: 0.24.0 + version: 0.28.0 pool: git: https://github.com/ysbaddaden/pool.git @@ -61,5 +61,5 @@ shards: sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git - version: 0.18.0 + version: 0.21.0 diff --git a/shard.yml b/shard.yml index db204551..2320556f 100644 --- a/shard.yml +++ b/shard.yml @@ -12,10 +12,10 @@ targets: dependencies: pg: github: will/crystal-pg - version: ~> 0.24.0 + version: ~> 0.28.0 sqlite3: github: crystal-lang/crystal-sqlite3 - version: ~> 0.18.0 + version: ~> 0.21.0 kemal: github: kemalcr/kemal version: ~> 1.1.2 diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 6cc30142..96400f47 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,78 +1,3 @@ -def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - object_inner_2 = { - "2:0:embedded" => { - "1:0:varint" => 0_i64, - }, - "5:varint" => 50_i64, - "6:varint" => 1_i64, - "7:varint" => (page * 30).to_i64, - "9:varint" => 1_i64, - "10:varint" => 0_i64, - } - - object_inner_2_encoded = object_inner_2 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - content_type_numerical = - case content_type - when "videos" then 15 - when "livestreams" then 14 - else 15 # Fallback to "videos" - end - - sort_by_numerical = - case sort_by - when "newest" then 1_i64 - when "popular" then 2_i64 - when "oldest" then 4_i64 - else 1_i64 # Fallback to "newest" - end - - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "#{content_type_numerical}:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, - }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "3:varint" => sort_by_numerical, - }, - }, - }, - } - - object_inner_1_encoded = object_inner_1 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:string" => object_inner_1_encoded, - "35:string" => "browse-feed#{ucid}videos102", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def make_initial_content_ctoken(ucid, content_type, sort_by) : String - return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by) -end - module Invidious::Channel::Tabs extend self @@ -101,7 +26,7 @@ module Invidious::Channel::Tabs end def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by) + continuation ||= make_initial_videos_ctoken(ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, author, ucid) @@ -130,14 +55,10 @@ module Invidious::Channel::Tabs # Shorts # ------------------- - def get_shorts(channel : AboutChannel, continuation : String? = nil) - if continuation.nil? - # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts" - # TODO: try to extract the continuation tokens that allows other sorting options - initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation: continuation) - end + def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_initial_shorts_ctoken(channel.ucid, sort_by) + initial_data = YoutubeAPI.browse(continuation: continuation) + return extract_items(initial_data, channel.author, channel.ucid) end @@ -145,9 +66,8 @@ module Invidious::Channel::Tabs # Livestreams # ------------------- - def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by) - + def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_initial_livestreams_ctoken(channel.ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, channel.author, channel.ucid) @@ -171,4 +91,102 @@ module Invidious::Channel::Tabs return items, next_continuation end + + # ------------------- + # C-tokens + # ------------------- + + private def sort_options_videos_short(sort_by : String) + case sort_by + when "newest" then return 4_i64 + when "popular" then return 2_i64 + when "oldest" then return 5_i64 + else return 4_i64 # Fallback to "newest" + end + end + + # Generate the initial "continuation token" to get the first page of the + # "videos" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_initial_videos_ctoken(ucid : String, sort_by = "newest") + object = { + "15:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "4:varint" => sort_options_videos_short(sort_by), + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # Generate the initial "continuation token" to get the first page of the + # "shorts" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_initial_shorts_ctoken(ucid : String, sort_by = "newest") + object = { + "10:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "4:varint" => sort_options_videos_short(sort_by), + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # Generate the initial "continuation token" to get the first page of the + # "livestreams" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_initial_livestreams_ctoken(ucid : String, sort_by = "newest") + sort_by_numerical = + case sort_by + when "newest" then 12_i64 + when "popular" then 14_i64 + when "oldest" then 13_i64 + else 12_i64 # Fallback to "newest" + end + + object = { + "14:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "5:varint" => sort_by_numerical, + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # The protobuf structure common between videos/shorts/livestreams + private def channel_ctoken_wrap(ucid : String, object) + object_inner = { + "110:embedded" => { + "3:embedded" => object, + }, + } + + object_inner_encoded = object_inner + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => object_inner_encoded, + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation + end end diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index 23a1aafc..1ba3ea61 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -1,8 +1,22 @@ +# Languages requiring a better level of translation (at least 20%) +# to be added to the list below: +# +# "af" => "", # Afrikaans +# "az" => "", # Azerbaijani +# "be" => "", # Belarusian +# "bn_BD" => "", # Bengali (Bangladesh) +# "ia" => "", # Interlingua +# "or" => "", # Odia +# "tk" => "", # Turkmen +# "tok => "", # Toki Pona +# LOCALES_LIST = { "ar" => "العربية", # Arabic + "bg" => "български", # Bulgarian "bn" => "বাংলা", # Bengali "ca" => "Català", # Catalan "cs" => "Čeština", # Czech + "cy" => "Cymraeg", # Welsh "da" => "Dansk", # Danish "de" => "Deutsch", # German "el" => "Ελληνικά", # Greek @@ -23,6 +37,7 @@ LOCALES_LIST = { "it" => "Italiano", # Italian "ja" => "日本語", # Japanese "ko" => "한국어", # Korean + "lmo" => "Lombard", # Lombard "lt" => "Lietuvių", # Lithuanian "nb-NO" => "Norsk bokmål", # Norwegian Bokmål "nl" => "Nederlands", # Dutch diff --git a/src/invidious/helpers/logger.cr b/src/invidious/helpers/logger.cr index 36a3a7f9..03349595 100644 --- a/src/invidious/helpers/logger.cr +++ b/src/invidious/helpers/logger.cr @@ -12,7 +12,9 @@ enum LogLevel end class Invidious::LogHandler < Kemal::BaseLogHandler - def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, @color : Bool = true) + def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true) + Colorize.enabled = use_color + Colorize.on_tty_only! end def call(context : HTTP::Server::Context) @@ -56,8 +58,7 @@ class Invidious::LogHandler < Kemal::BaseLogHandler {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) if LogLevel::{{level.id.capitalize}} >= @level - puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})).toggle(@color)) - + puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}}))) end end {% end %} diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr index 2922b060..59a30745 100644 --- a/src/invidious/routes/api/v1/search.cr +++ b/src/invidious/routes/api/v1/search.cr @@ -31,9 +31,7 @@ module Invidious::Routes::API::V1::Search query = env.params.query["q"]? || "" begin - client = HTTP::Client.new("suggestqueries-clients6.youtube.com") - client.before_request { |r| add_yt_headers(r) } - + client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true) url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt" response = client.get(url).body diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index ce158c33..927ad636 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -20,10 +20,11 @@ module Invidious::Routes::Channels sort_by = env.params.query["sort_by"]?.try &.downcase if channel.auto_generated + sort_by ||= "last" sort_options = {"last", "oldest", "newest"} items, next_continuation = fetch_channel_playlists( - channel.ucid, channel.author, continuation, (sort_by || "last") + channel.ucid, channel.author, continuation, sort_by ) items.uniq! do |item| @@ -49,9 +50,11 @@ module Invidious::Routes::Channels end next_continuation = nil else + sort_by ||= "newest" sort_options = {"newest", "oldest", "popular"} - items, next_continuation = Channel::Tabs.get_videos( - channel, continuation: continuation, sort_by: (sort_by || "newest") + + items, next_continuation = Channel::Tabs.get_60_videos( + channel, continuation: continuation, sort_by: sort_by ) end end @@ -82,13 +85,12 @@ module Invidious::Routes::Channels end next_continuation = nil else - # TODO: support sort option for shorts - sort_by = "" - sort_options = [] of String + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} # Fetch items and continuation token items, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 980ca976..d301d2e5 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -47,11 +47,7 @@ module Invidious::Routes::VideoPlayback headers["Range"] = "bytes=#{range_for_head}" end - headers["Origin"] = "https://www.youtube.com" - headers["Referer"] = "https://www.youtube.com/" - headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" - - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) response = HTTP::Client::Response.new(500) error = "" 5.times do @@ -66,7 +62,7 @@ module Invidious::Routes::VideoPlayback if new_host != host host = new_host client.close - client = make_client(URI.parse(new_host), region, force_resolve = true) + client = make_client(URI.parse(new_host), region, force_resolve: true) end url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}" @@ -80,7 +76,7 @@ module Invidious::Routes::VideoPlayback fvip = "3" host = "https://r#{fvip}---#{mn}.googlevideo.com" - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) rescue ex error = ex.message end @@ -205,7 +201,7 @@ module Invidious::Routes::VideoPlayback break else client.close - client = make_client(URI.parse(host), region, force_resolve = true) + client = make_client(URI.parse(host), region, force_resolve: true) end end diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index c7c4c675..c4a73aa7 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -22,12 +22,8 @@ struct YoutubeConnectionPool response = yield conn rescue ex conn.close + conn = make_client(url, force_resolve: true) - conn = HTTP::Client.new(url) - conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy - conn.family = CONFIG.force_resolve - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" response = yield conn ensure pool.release(conn) @@ -37,12 +33,15 @@ struct YoutubeConnectionPool end private def build_pool - DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - conn = HTTP::Client.new(url) - conn.family = CONFIG.force_resolve - conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC - conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" - conn + options = DB::Pool::Options.new( + initial_pool_size: 0, + max_pool_size: capacity, + max_idle_pool_size: capacity, + checkout_timeout: timeout + ) + + DB::Pool(HTTP::Client).new(options) do + next make_client(url, force_resolve: true) end end end @@ -62,15 +61,17 @@ def add_yt_headers(request) end end -def make_client(url : URI, region = nil, force_resolve : Bool = false) +def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false) client = HTTP::Client.new(url) + client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy # Force the usage of a specific configured IP Family if force_resolve client.family = CONFIG.force_resolve + client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC end - client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" + client.before_request { |r| add_yt_headers(r) } if url.host.try &.ends_with?("youtube.com") || force_youtube_headers client.read_timeout = 10.seconds client.connect_timeout = 10.seconds @@ -78,7 +79,7 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) end def make_client(url : URI, region = nil, force_resolve : Bool = false, &) - client = make_client(url, region, force_resolve) + client = make_client(url, region, force_resolve: force_resolve) begin yield client ensure diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 4074de86..2631b62a 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -21,6 +21,7 @@ private ITEM_PARSERS = { Parsers::ItemSectionRendererParser, Parsers::ContinuationItemRendererParser, Parsers::HashtagRendererParser, + Parsers::LockupViewModelParser, } private alias InitialData = Hash(String, JSON::Any) @@ -467,9 +468,9 @@ private module Parsers # Parses an InnerTube richItemRenderer into a SearchVideo. # Returns nil when the given object isn't a RichItemRenderer # - # A richItemRenderer seems to be a simple wrapper for a videoRenderer, used - # by the result page for hashtags and for the podcast tab on channels. - # It is located inside a continuationItems container for hashtags. + # A richItemRenderer seems to be a simple wrapper for a various other types, + # used on the hashtags result page and the channel podcast tab. It is located + # itself inside a richGridRenderer container. # module RichItemRendererParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) @@ -482,6 +483,8 @@ private module Parsers child = VideoRendererParser.process(item_contents, author_fallback) child ||= ReelItemRendererParser.process(item_contents, author_fallback) child ||= PlaylistRendererParser.process(item_contents, author_fallback) + child ||= LockupViewModelParser.process(item_contents, author_fallback) + child ||= ShortsLockupViewModelParser.process(item_contents, author_fallback) return child end @@ -496,6 +499,9 @@ private module Parsers # reelItemRenderer items are used in the new (2022) channel layout, # in the "shorts" tab. # + # NOTE: As of 10/2024, it might have been fully replaced by shortsLockupViewModel + # TODO: Confirm that hypothesis + # module ReelItemRendererParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["reelItemRenderer"]? @@ -582,6 +588,135 @@ private module Parsers end end + # Parses an InnerTube lockupViewModel into a SearchPlaylist. + # Returns nil when the given object is not a lockupViewModel. + # + # This structure is present since November 2024 on the "podcasts" and + # "playlists" tabs of the channel page. It is usually encapsulated in either + # a richItemRenderer or a richGridRenderer. + # + module LockupViewModelParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["lockupViewModel"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + playlist_id = item_contents["contentId"].as_s + + thumbnail_view_model = item_contents.dig( + "contentImage", "collectionThumbnailViewModel", + "primaryThumbnail", "thumbnailViewModel" + ) + + thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s + + # This complicated sequences tries to extract the following data structure: + # "overlays": [{ + # "thumbnailOverlayBadgeViewModel": { + # "thumbnailBadges": [{ + # "thumbnailBadgeViewModel": { + # "text": "430 episodes", + # "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT" + # } + # }] + # } + # }] + # + # NOTE: this simplistic `.to_i` conversion might not work on larger + # playlists and hasn't been tested. + video_count = thumbnail_view_model.dig("overlays").as_a + .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a) + .flatten + .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try { |node| + {"episodes", "videos"}.any? { |str| node.as_s.ends_with?(str) } + }) + .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false) + + metadata = item_contents.dig("metadata", "lockupMetadataViewModel") + title = metadata.dig("title", "content").as_s + + # TODO: Retrieve "updated" info from metadata parts + # rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a + # parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s) + # One of these parts should contain a string like: "Updated 2 days ago" + + # TODO: Maybe add a button to access the first video of the playlist? + # item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint") + # Available fields: "videoId", "playlistId", "params" + + return SearchPlaylist.new({ + title: title, + id: playlist_id, + author: author_fallback.name, + ucid: author_fallback.id, + video_count: video_count || -1, + videos: [] of SearchPlaylistVideo, + thumbnail: thumbnail, + author_verified: false, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + + # Parses an InnerTube shortsLockupViewModel into a SearchVideo. + # Returns nil when the given object is not a shortsLockupViewModel. + # + # This structure is present since around October 2024 on the "shorts" tab of + # the channel page and likely replaces the reelItemRenderer structure. It is + # usually (always?) encapsulated in a richItemRenderer. + # + module ShortsLockupViewModelParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["shortsLockupViewModel"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + # TODO: Maybe add support for "oardefault.jpg" thumbnails? + # thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s + # Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?... + + video_id = item_contents.dig( + "onTap", "innertubeCommand", "reelWatchEndpoint", "videoId" + ).as_s + + title = item_contents.dig("overlayMetadata", "primaryText", "content").as_s + + view_count = short_text_to_number( + item_contents.dig("overlayMetadata", "secondaryText", "content").as_s + ) + + # Approximate to one minute, as "shorts" generally don't exceed that. + # NOTE: The actual duration is not provided by Youtube anymore. + # TODO: Maybe use -1 as an error value and handle that on the frontend? + duration = 60_i32 + + SearchVideo.new({ + title: title, + id: video_id, + author: author_fallback.name, + ucid: author_fallback.id, + published: Time.unix(0), + views: view_count, + description_html: "", + length_seconds: duration, + premiere_timestamp: Time.unix(0), + author_verified: false, + badges: VideoBadges::None, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + # Parses an InnerTube continuationItemRenderer into a Continuation. # Returns nil when the given object isn't a continuationItemRenderer. # diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index b2760532..366f6b80 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -639,6 +639,11 @@ module YoutubeAPI # Send the POST request body = YT_POOL.client() do |client| client.post(url, headers: headers, body: data.to_json) do |response| + if response.status_code != 200 + raise InfoException.new("Error: non 200 status code. Youtube API returned \ + status code #{response.status_code}. See \ + https://docs.invidious.io/youtube-errors-explained/ for troubleshooting.") + end self._decompress(response.body_io, response.headers["Content-Encoding"]?) end end