From 9cd2e93a2ee8f2f0f570bcb8fbe584f5c502a34e Mon Sep 17 00:00:00 2001 From: thansk <53181514+thansk@users.noreply.github.com> Date: Sun, 19 May 2024 11:46:55 +0000 Subject: [PATCH 01/30] feat: allow submitting search with mouse --- assets/css/default.css | 19 ++++++++++++++++++- src/invidious/views/components/search_box.ecr | 1 + 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/assets/css/default.css b/assets/css/default.css index a47762ec..d86ec7bc 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -278,7 +278,14 @@ div.thumbnail > .bottom-right-overlay { display: inline; } -.searchbar .pure-form fieldset { padding: 0; } +.searchbar .pure-form { + display: flex; +} + +.searchbar .pure-form fieldset { + padding: 0; + flex: 1; +} .searchbar input[type="search"] { width: 100%; @@ -310,6 +317,16 @@ input[type="search"]::-webkit-search-cancel-button { background-size: 14px; } +.searchbar #searchbutton { + border: 0; + background: none; + text-transform: uppercase; +} + +.searchbar #searchbutton:hover { + color: rgb(0, 182, 240); +} + .user-field { display: flex; flex-direction: row; diff --git a/src/invidious/views/components/search_box.ecr b/src/invidious/views/components/search_box.ecr index a03785d1..c5488255 100644 --- a/src/invidious/views/components/search_box.ecr +++ b/src/invidious/views/components/search_box.ecr @@ -6,4 +6,5 @@ title="<%= translate(locale, "search") %>" value="<%= env.get?("search").try {|x| HTML.escape(x.as(String)) } %>"> + From 5abafb8296330dfc7fe7ab630661e0cc8e04ef85 Mon Sep 17 00:00:00 2001 From: thansk <53181514+thansk@users.noreply.github.com> Date: Mon, 20 May 2024 11:49:56 +0000 Subject: [PATCH 02/30] fix: use a search icon instead of text --- assets/css/default.css | 3 +++ src/invidious/views/components/search_box.ecr | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/assets/css/default.css b/assets/css/default.css index d86ec7bc..20ec3222 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -321,6 +321,9 @@ input[type="search"]::-webkit-search-cancel-button { border: 0; background: none; text-transform: uppercase; + display: grid; + place-items: center; + width: 1.5em; } .searchbar #searchbutton:hover { diff --git a/src/invidious/views/components/search_box.ecr b/src/invidious/views/components/search_box.ecr index c5488255..b679b031 100644 --- a/src/invidious/views/components/search_box.ecr +++ b/src/invidious/views/components/search_box.ecr @@ -6,5 +6,9 @@ title="<%= translate(locale, "search") %>" value="<%= env.get?("search").try {|x| HTML.escape(x.as(String)) } %>"> - + From 1ce2d10c505a7e0c3972acfb626a0ae3c9af3d57 Mon Sep 17 00:00:00 2001 From: thansk <53181514+thansk@users.noreply.github.com> Date: Mon, 20 May 2024 14:17:30 +0000 Subject: [PATCH 03/30] fix: use ion icon for search icon --- assets/css/default.css | 7 ++----- src/invidious/views/components/search_box.ecr | 4 +--- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/assets/css/default.css b/assets/css/default.css index 20ec3222..1445f65f 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -318,12 +318,9 @@ input[type="search"]::-webkit-search-cancel-button { } .searchbar #searchbutton { - border: 0; + border: none; background: none; - text-transform: uppercase; - display: grid; - place-items: center; - width: 1.5em; + margin-top: 0; } .searchbar #searchbutton:hover { diff --git a/src/invidious/views/components/search_box.ecr b/src/invidious/views/components/search_box.ecr index b679b031..29da2c52 100644 --- a/src/invidious/views/components/search_box.ecr +++ b/src/invidious/views/components/search_box.ecr @@ -7,8 +7,6 @@ value="<%= env.get?("search").try {|x| HTML.escape(x.as(String)) } %>"> From 96ade642faad7052b0b70171a2c0ac4c09819151 Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Sun, 26 Nov 2023 20:24:04 -0500 Subject: [PATCH 04/30] Channel: Render age restricted channels --- src/invidious/channels/about.cr | 170 +++++++++++++----------- src/invidious/playlists.cr | 7 + src/invidious/routes/api/v1/channels.cr | 89 ++++++++++--- src/invidious/routes/channels.cr | 74 ++++++++--- 4 files changed, 221 insertions(+), 119 deletions(-) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index edaf5c12..b3561fcd 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -15,7 +15,8 @@ record AboutChannel, allowed_regions : Array(String), tabs : Array(String), tags : Array(String), - verified : Bool + verified : Bool, + is_age_gated : Bool def get_about_info(ucid, locale) : AboutChannel begin @@ -45,46 +46,102 @@ def get_about_info(ucid, locale) : AboutChannel end tags = [] of String + tab_names = [] of String + total_views = 0_i64 + joined = Time.unix(0) - if auto_generated - author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s - author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s - author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s - - # Raises a KeyError on failure. - banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banner = banners.try &.[-1]?.try &.["url"].as_s? - - description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] - # some channels have the description in a simpleText - # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ - description_node = description_base_node.dig?("simpleText") || description_base_node - - tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") - .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String + if ageGate = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer") + description_node = nil + author = ageGate["channelTitle"].as_s + ucid = initdata["responseContext"]["serviceTrackingParams"][0]["params"][0]["value"].as_s + author_url = "https://www.youtube.com/channel/" + ucid + author_thumbnail = ageGate["avatar"]["thumbnails"][0]["url"].as_s + banner = nil + is_family_friendly = false + is_age_gated = true + tab_names = ["videos", "shorts", "streams"] + auto_generated = false else - author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s - author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s - author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s - author_verified = has_verified_badge?(initdata.dig?("header", "c4TabbedHeaderRenderer", "badges")) + if auto_generated + author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s + author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s + author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s - ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s + # Raises a KeyError on failure. + banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banner = banners.try &.[-1]?.try &.["url"].as_s? - # Raises a KeyError on failure. - banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources") - banner = banners.try &.[-1]?.try &.["url"].as_s? + description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] + # some channels have the description in a simpleText + # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ + description_node = description_base_node.dig?("simpleText") || description_base_node - # if banner.includes? "channels/c4/default_banner" - # banner = nil - # end + tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") + .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String + else + author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s + author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s + author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s + author_verified = has_verified_badge?(initdata.dig?("header", "c4TabbedHeaderRenderer", "badges")) - description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? - tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String + ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s + + # Raises a KeyError on failure. + banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? + banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources") + banner = banners.try &.[-1]?.try &.["url"].as_s? + + # if banner.includes? "channels/c4/default_banner" + # banner = nil + # end + + description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? + tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String + end + + is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool + if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? + # Get the name of the tabs available on this channel + tab_names = tabs_json.as_a.compact_map do |entry| + name = entry.dig?("tabRenderer", "title").try &.as_s.downcase + + # This is a small fix to not add extra code on the HTML side + # I.e, the URL for the "live" tab is .../streams, so use "streams" + # everywhere for the sake of simplicity + (name == "live") ? "streams" : name + end + + # Get the currently active tab ("About") + about_tab = extract_selected_tab(tabs_json) + + # Try to find the about metadata section + channel_about_meta = about_tab.dig?( + "content", + "sectionListRenderer", "contents", 0, + "itemSectionRenderer", "contents", 0, + "channelAboutFullMetadataRenderer" + ) + + if !channel_about_meta.nil? + total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 + + # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. + joined = extract_text(channel_about_meta["joinedDateText"]?) + .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) + + # Normal Auto-generated channels + # https://support.google.com/youtube/answer/2579942 + # For auto-generated channels, channel_about_meta only has + # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] + auto_generated = ( + (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ + extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" || + channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube" + ) + end + end end - is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool - allowed_regions = initdata .dig?("microformat", "microformatDataRenderer", "availableCountries") .try &.as_a.map(&.as_s) || [] of String @@ -102,52 +159,6 @@ def get_about_info(ucid, locale) : AboutChannel end end - total_views = 0_i64 - joined = Time.unix(0) - - tab_names = [] of String - - if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? - # Get the name of the tabs available on this channel - tab_names = tabs_json.as_a.compact_map do |entry| - name = entry.dig?("tabRenderer", "title").try &.as_s.downcase - - # This is a small fix to not add extra code on the HTML side - # I.e, the URL for the "live" tab is .../streams, so use "streams" - # everywhere for the sake of simplicity - (name == "live") ? "streams" : name - end - - # Get the currently active tab ("About") - about_tab = extract_selected_tab(tabs_json) - - # Try to find the about metadata section - channel_about_meta = about_tab.dig?( - "content", - "sectionListRenderer", "contents", 0, - "itemSectionRenderer", "contents", 0, - "channelAboutFullMetadataRenderer" - ) - - if !channel_about_meta.nil? - total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 - - # The joined text is split to several sub strings. The reduce joins those strings before parsing the date. - joined = extract_text(channel_about_meta["joinedDateText"]?) - .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) - - # Normal Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - # For auto-generated channels, channel_about_meta only has - # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"] - auto_generated = ( - (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \ - extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" || - channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube" - ) - end - end - sub_count = 0 if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a) @@ -177,6 +188,7 @@ def get_about_info(ucid, locale) : AboutChannel tabs: tab_names, tags: tags, verified: author_verified || false, + is_age_gated: is_age_gated || false, ) end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index a227f794..3e6eef95 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -46,8 +46,14 @@ struct PlaylistVideo XML.build { |xml| to_xml(xml) } end + def to_json(locale : String?, json : JSON::Builder) + to_json(json) + end + def to_json(json : JSON::Builder, index : Int32? = nil) json.object do + json.field "type", "video" + json.field "title", self.title json.field "videoId", self.id @@ -67,6 +73,7 @@ struct PlaylistVideo end json.field "lengthSeconds", self.length_seconds + json.field "liveNow", self.live_now end end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 43a5c35b..2da76134 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -27,10 +27,21 @@ module Invidious::Routes::API::V1::Channels # Retrieve "sort by" setting from URL parameters sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" - begin - videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) + rescue ex + return error_json(500, ex) + end end JSON.build do |json| @@ -84,6 +95,7 @@ module Invidious::Routes::API::V1::Channels json.field "joined", channel.joined.to_unix json.field "autoGenerated", channel.auto_generated + json.field "ageGated", channel.is_age_gated json.field "isFamilyFriendly", channel.is_family_friendly json.field "description", html_to_content(channel.description_html) json.field "descriptionHtml", channel.description_html @@ -142,12 +154,23 @@ module Invidious::Routes::API::V1::Channels sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_60_videos( - channel, continuation: continuation, sort_by: sort_by - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_60_videos( + channel, continuation: continuation, sort_by: sort_by + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| @@ -176,12 +199,23 @@ module Invidious::Routes::API::V1::Channels # Retrieve continuation from URL parameters continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| @@ -211,12 +245,23 @@ module Invidious::Routes::API::V1::Channels sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? - begin - videos, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation, sort_by: sort_by - ) - rescue ex - return error_json(500, ex) + if channel.is_age_gated + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULV")) + videos = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + videos = [] of PlaylistVideo + end + next_continuation = nil + else + begin + videos, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation, sort_by: sort_by + ) + rescue ex + return error_json(500, ex) + end end return JSON.build do |json| diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 360af2cd..952098e0 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -36,12 +36,24 @@ module Invidious::Routes::Channels items = items.select(SearchPlaylist) items.each(&.author = "") else - sort_options = {"newest", "oldest", "popular"} - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_videos( - channel, continuation: continuation, sort_by: (sort_by || "newest") - ) + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULF")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + sort_options = {"newest", "oldest", "popular"} + items, next_continuation = Channel::Tabs.get_videos( + channel, continuation: continuation, sort_by: (sort_by || "newest") + ) + end end selected_tab = Frontend::ChannelPage::TabsAvailable::Videos @@ -58,14 +70,27 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - # TODO: support sort option for shorts - sort_by = "" - sort_options = [] of String + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + # TODO: support sort option for shorts + sort_by = "" + sort_options = [] of String - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation - ) + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_shorts( + channel, continuation: continuation + ) + end selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts templated "channel" @@ -81,13 +106,26 @@ module Invidious::Routes::Channels return env.redirect "/channel/#{channel.ucid}" end - sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" - sort_options = {"newest", "oldest", "popular"} + if channel.is_age_gated + sort_by = "" + sort_options = [] of String + begin + playlist = get_playlist(channel.ucid.sub("UC", "UULV")) + items = get_playlist_videos(playlist, offset: 0) + rescue ex : InfoException + # playlist doesnt exist. + items = [] of PlaylistVideo + end + next_continuation = nil + else + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} - # Fetch items and continuation token - items, next_continuation = Channel::Tabs.get_60_livestreams( - channel, continuation: continuation, sort_by: sort_by - ) + # Fetch items and continuation token + items, next_continuation = Channel::Tabs.get_60_livestreams( + channel, continuation: continuation, sort_by: sort_by + ) + end selected_tab = Frontend::ChannelPage::TabsAvailable::Streams templated "channel" From e31053e812517d8d097368ae8863404a4a563731 Mon Sep 17 00:00:00 2001 From: ChunkyProgrammer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Sun, 19 May 2024 10:46:05 -0400 Subject: [PATCH 05/30] Use dig to get properties Co-authored-by: Samantaz Fox --- src/invidious/channels/about.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index b3561fcd..1380044a 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -53,9 +53,9 @@ def get_about_info(ucid, locale) : AboutChannel if ageGate = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer") description_node = nil author = ageGate["channelTitle"].as_s - ucid = initdata["responseContext"]["serviceTrackingParams"][0]["params"][0]["value"].as_s - author_url = "https://www.youtube.com/channel/" + ucid - author_thumbnail = ageGate["avatar"]["thumbnails"][0]["url"].as_s + ucid = initdata.dig("responseContext", "serviceTrackingParams", 0, "params", 0, "value").as_s + author_url = "https://www.youtube.com/channel/#{ucid}" + author_thumbnail = ageGate.dig("avatar", "thumbnails", 0, "url").as_s banner = nil is_family_friendly = false is_age_gated = true From 466bfbb30637b625ceda1e1073dbc190e51c8dc9 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 14 Aug 2024 21:43:37 +0200 Subject: [PATCH 06/30] SigHelper: Fix inverted time comparison in 'check_update' --- src/invidious/helpers/signatures.cr | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr index 84a8a86d..82a28fc0 100644 --- a/src/invidious/helpers/signatures.cr +++ b/src/invidious/helpers/signatures.cr @@ -10,10 +10,8 @@ class Invidious::DecryptFunction end def check_update - now = Time.utc - # If we have updated in the last 5 minutes, do nothing - return if (now - @last_update) > 5.minutes + return if (Time.utc - @last_update) < 5.minutes # Get the amount of time elapsed since when the player was updated, in the # event where multiple invidious processes are run in parallel. From acbb62586611ec8fd25df9b56f2042a830933155 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 15 Aug 2024 12:54:43 +0200 Subject: [PATCH 07/30] YtAPI: Update clients to latest version --- src/invidious/yt_backend/youtube_api.cr | 26 ++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 6d585bf2..d66bf7aa 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -6,10 +6,10 @@ module YoutubeAPI extend self # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history - private ANDROID_APP_VERSION = "19.14.42" - private ANDROID_USER_AGENT = "com.google.android.youtube/19.14.42 (Linux; U; Android 12; US) gzip" - private ANDROID_SDK_VERSION = 31_i64 + private ANDROID_APP_VERSION = "19.32.34" private ANDROID_VERSION = "12" + private ANDROID_USER_AGENT = "com.google.android.youtube/#{ANDROID_APP_VERSION} (Linux; U; Android #{ANDROID_VERSION}; US) gzip" + private ANDROID_SDK_VERSION = 31_i64 private ANDROID_TS_APP_VERSION = "1.9" private ANDROID_TS_USER_AGENT = "com.google.android.youtube/1.9 (Linux; U; Android 12; US) gzip" @@ -17,9 +17,9 @@ module YoutubeAPI # For Apple device names, see https://gist.github.com/adamawolf/3048717 # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases, # then go to the dedicated article of the major version you want. - private IOS_APP_VERSION = "19.16.3" - private IOS_USER_AGENT = "com.google.ios.youtube/19.16.3 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)" - private IOS_VERSION = "17.4.0.21E219" # Major.Minor.Patch.Build + private IOS_APP_VERSION = "19.32.8" + private IOS_USER_AGENT = "com.google.ios.youtube/#{IOS_APP_VERSION} (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)" + private IOS_VERSION = "17.6.1.21G93" # Major.Minor.Patch.Build private WINDOWS_VERSION = "10.0" @@ -48,7 +48,7 @@ module YoutubeAPI ClientType::Web => { name: "WEB", name_proto: "1", - version: "2.20240304.00.00", + version: "2.20240814.00.00", screen: "WATCH_FULL_SCREEN", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -57,7 +57,7 @@ module YoutubeAPI ClientType::WebEmbeddedPlayer => { name: "WEB_EMBEDDED_PLAYER", name_proto: "56", - version: "1.20240303.00.00", + version: "1.20240812.01.00", screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -66,7 +66,7 @@ module YoutubeAPI ClientType::WebMobile => { name: "MWEB", name_proto: "2", - version: "2.20240304.08.00", + version: "2.20240813.02.00", os_name: "Android", os_version: ANDROID_VERSION, platform: "MOBILE", @@ -74,7 +74,7 @@ module YoutubeAPI ClientType::WebScreenEmbed => { name: "WEB", name_proto: "1", - version: "2.20240304.00.00", + version: "2.20240814.00.00", screen: "EMBED", os_name: "Windows", os_version: WINDOWS_VERSION, @@ -147,8 +147,8 @@ module YoutubeAPI ClientType::IOSMusic => { name: "IOS_MUSIC", name_proto: "26", - version: "6.42", - user_agent: "com.google.ios.youtubemusic/6.42 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)", + version: "7.14", + user_agent: "com.google.ios.youtubemusic/7.14 (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)", device_make: "Apple", device_model: "iPhone14,5", os_name: "iPhone", @@ -161,7 +161,7 @@ module YoutubeAPI ClientType::TvHtml5 => { name: "TVHTML5", name_proto: "7", - version: "7.20240304.10.00", + version: "7.20240813.07.00", }, ClientType::TvHtml5ScreenEmbed => { name: "TVHTML5_SIMPLY_EMBEDDED_PLAYER", From cc33d3f074c24be8b9afac5ddbc0465a87f0d867 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 15 Aug 2024 18:13:41 +0200 Subject: [PATCH 08/30] YtAPI: Also update User-Agent string --- src/invidious/yt_backend/connection_pool.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 0ac785e6..ca612083 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,6 +1,6 @@ def add_yt_headers(request) request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" From 0b28054f8ac4066d5f2966a75a92eb935247d737 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 15 Aug 2024 18:26:17 +0200 Subject: [PATCH 09/30] videos: Fix XSS vulnerability in description/comments Patch provided by e-mail, thanks to an anonymous user whose cats are named Yoshi and Yasuo. Comment is mine --- src/invidious/videos/description.cr | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr index c7191dec..1371bebb 100644 --- a/src/invidious/videos/description.cr +++ b/src/invidious/videos/description.cr @@ -36,7 +36,13 @@ def parse_description(desc, video_id : String) : String? return "" if content.empty? commands = desc["commandRuns"]?.try &.as_a - return content if commands.nil? + if commands.nil? + # Slightly faster than HTML.escape, as we're only doing one pass on + # the string instead of five for the standard library + return String.build do |str| + copy_string(str, content.each_codepoint, content.size) + end + end # Not everything is stored in UTF-8 on youtube's side. The SMP codepoints # (0x10000 and above) are encoded as UTF-16 surrogate pairs, which are From 6878822c4d621bc2a2ba65c117efc65246e9a1ca Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 8 Oct 2023 12:58:04 +0200 Subject: [PATCH 10/30] Storyboards: Move parser to its own file --- src/invidious/videos.cr | 61 +------------------------- src/invidious/videos/storyboard.cr | 69 ++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 59 deletions(-) create mode 100644 src/invidious/videos/storyboard.cr diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 6d0cf9ba..73321909 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -177,65 +177,8 @@ struct Video # Misc. methods def storyboards - storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec") - .try &.as_s.split("|") - - if !storyboards - if storyboard = info.dig?("storyboards", "playerLiveStoryboardSpecRenderer", "spec").try &.as_s - return [{ - url: storyboard.split("#")[0], - width: 106, - height: 60, - count: -1, - interval: 5000, - storyboard_width: 3, - storyboard_height: 3, - storyboard_count: -1, - }] - end - end - - items = [] of NamedTuple( - url: String, - width: Int32, - height: Int32, - count: Int32, - interval: Int32, - storyboard_width: Int32, - storyboard_height: Int32, - storyboard_count: Int32) - - return items if !storyboards - - url = URI.parse(storyboards.shift) - params = HTTP::Params.parse(url.query || "") - - storyboards.each_with_index do |sb, i| - width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#") - params["sigh"] = sigh - url.query = params.to_s - - width = width.to_i - height = height.to_i - count = count.to_i - interval = interval.to_i - storyboard_width = storyboard_width.to_i - storyboard_height = storyboard_height.to_i - storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i - - items << { - url: url.to_s.sub("$L", i).sub("$N", "M$M"), - width: width, - height: height, - count: count, - interval: interval, - storyboard_width: storyboard_width, - storyboard_height: storyboard_height, - storyboard_count: storyboard_count, - } - end - - items + container = info.dig?("storyboards") || JSON::Any.new("{}") + return IV::Videos::Storyboard.from_yt_json(container) end def paid diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr new file mode 100644 index 00000000..b4302d88 --- /dev/null +++ b/src/invidious/videos/storyboard.cr @@ -0,0 +1,69 @@ +require "uri" +require "http/params" + +module Invidious::Videos + struct Storyboard + # Parse the JSON structure from Youtube + def self.from_yt_json(container : JSON::Any) + storyboards = container.dig?("playerStoryboardSpecRenderer", "spec") + .try &.as_s.split("|") + + if !storyboards + if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s + return [{ + url: storyboard.split("#")[0], + width: 106, + height: 60, + count: -1, + interval: 5000, + storyboard_width: 3, + storyboard_height: 3, + storyboard_count: -1, + }] + end + end + + items = [] of NamedTuple( + url: String, + width: Int32, + height: Int32, + count: Int32, + interval: Int32, + storyboard_width: Int32, + storyboard_height: Int32, + storyboard_count: Int32) + + return items if !storyboards + + url = URI.parse(storyboards.shift) + params = HTTP::Params.parse(url.query || "") + + storyboards.each_with_index do |sb, i| + width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#") + params["sigh"] = sigh + url.query = params.to_s + + width = width.to_i + height = height.to_i + count = count.to_i + interval = interval.to_i + storyboard_width = storyboard_width.to_i + storyboard_height = storyboard_height.to_i + storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i + + items << { + url: url.to_s.sub("$L", i).sub("$N", "M$M"), + width: width, + height: height, + count: count, + interval: interval, + storyboard_width: storyboard_width, + storyboard_height: storyboard_height, + storyboard_count: storyboard_count, + } + end + + items + end + end +end From 8327862697774cd8076335fe2002875dd8c5a84a Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 8 Oct 2023 20:09:38 +0200 Subject: [PATCH 11/30] Storyboards: Use replace the NamedTuple by a struct --- src/invidious/jsonify/api_v1/video_json.cr | 18 +++---- src/invidious/routes/api/v1/videos.cr | 19 +++---- src/invidious/videos/storyboard.cr | 61 ++++++++++++---------- 3 files changed, 53 insertions(+), 45 deletions(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 59714828..44a34b18 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -273,15 +273,15 @@ module Invidious::JSONify::APIv1 json.array do storyboards.each do |storyboard| json.object do - json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}" - json.field "templateUrl", storyboard[:url] - json.field "width", storyboard[:width] - json.field "height", storyboard[:height] - json.field "count", storyboard[:count] - json.field "interval", storyboard[:interval] - json.field "storyboardWidth", storyboard[:storyboard_width] - json.field "storyboardHeight", storyboard[:storyboard_height] - json.field "storyboardCount", storyboard[:storyboard_count] + json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard.width}&height=#{storyboard.height}" + json.field "templateUrl", storyboard.url + json.field "width", storyboard.width + json.field "height", storyboard.height + json.field "count", storyboard.count + json.field "interval", storyboard.interval + json.field "storyboardWidth", storyboard.storyboard_width + json.field "storyboardHeight", storyboard.storyboard_height + json.field "storyboardCount", storyboard.storyboard_count end end end diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 42282f44..78f91a2e 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -205,7 +205,7 @@ module Invidious::Routes::API::V1::Videos env.response.content_type = "text/vtt" - storyboard = storyboards.select { |sb| width == "#{sb[:width]}" || height == "#{sb[:height]}" } + storyboard = storyboards.select { |sb| width == "#{sb.width}" || height == "#{sb.height}" } if storyboard.empty? haltf env, 404 @@ -215,21 +215,22 @@ module Invidious::Routes::API::V1::Videos WebVTT.build do |vtt| start_time = 0.milliseconds - end_time = storyboard[:interval].milliseconds + end_time = storyboard.interval.milliseconds - storyboard[:storyboard_count].times do |i| - url = storyboard[:url] + storyboard.storyboard_count.times do |i| + url = storyboard.url authority = /(i\d?).ytimg.com/.match!(url)[1]? + url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") url = "#{HOST_URL}/sb/#{authority}/#{url}" - storyboard[:storyboard_height].times do |j| - storyboard[:storyboard_width].times do |k| - current_cue_url = "#{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]}" + storyboard.storyboard_height.times do |j| + storyboard.storyboard_width.times do |k| + current_cue_url = "#{url}#xywh=#{storyboard.width * k},#{storyboard.height * j},#{storyboard.width - 2},#{storyboard.height}" vtt.cue(start_time, end_time, current_cue_url) - start_time += storyboard[:interval].milliseconds - end_time += storyboard[:interval].milliseconds + start_time += storyboard.interval.milliseconds + end_time += storyboard.interval.milliseconds end end end diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr index b4302d88..797fba12 100644 --- a/src/invidious/videos/storyboard.cr +++ b/src/invidious/videos/storyboard.cr @@ -3,6 +3,21 @@ require "http/params" module Invidious::Videos struct Storyboard + getter url : String + getter width : Int32 + getter height : Int32 + getter count : Int32 + getter interval : Int32 + getter storyboard_width : Int32 + getter storyboard_height : Int32 + getter storyboard_count : Int32 + + def initialize( + *, @url, @width, @height, @count, @interval, + @storyboard_width, @storyboard_height, @storyboard_count + ) + end + # Parse the JSON structure from Youtube def self.from_yt_json(container : JSON::Any) storyboards = container.dig?("playerStoryboardSpecRenderer", "spec") @@ -10,28 +25,20 @@ module Invidious::Videos if !storyboards if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s - return [{ - url: storyboard.split("#")[0], - width: 106, - height: 60, - count: -1, - interval: 5000, - storyboard_width: 3, + return [Storyboard.new( + url: storyboard.split("#")[0], + width: 106, + height: 60, + count: -1, + interval: 5000, + storyboard_width: 3, storyboard_height: 3, - storyboard_count: -1, - }] + storyboard_count: -1, + )] end end - items = [] of NamedTuple( - url: String, - width: Int32, - height: Int32, - count: Int32, - interval: Int32, - storyboard_width: Int32, - storyboard_height: Int32, - storyboard_count: Int32) + items = [] of Storyboard return items if !storyboards @@ -51,16 +58,16 @@ module Invidious::Videos storyboard_height = storyboard_height.to_i storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i - items << { - url: url.to_s.sub("$L", i).sub("$N", "M$M"), - width: width, - height: height, - count: count, - interval: interval, - storyboard_width: storyboard_width, + items << Storyboard.new( + url: url.to_s.sub("$L", i).sub("$N", "M$M"), + width: width, + height: height, + count: count, + interval: interval, + storyboard_width: storyboard_width, storyboard_height: storyboard_height, - storyboard_count: storyboard_count, - } + storyboard_count: storyboard_count + ) end items From da3d58f03c9b1617f96f4caf1e348a35105dd79c Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 8 Oct 2023 20:29:41 +0200 Subject: [PATCH 12/30] Storyboards: Cleanup and document code --- src/invidious/jsonify/api_v1/video_json.cr | 20 ++-- src/invidious/routes/api/v1/videos.cr | 52 +++++----- src/invidious/videos/storyboard.cr | 114 ++++++++++++++------- 3 files changed, 113 insertions(+), 73 deletions(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 44a34b18..4d12a072 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -271,17 +271,17 @@ module Invidious::JSONify::APIv1 def storyboards(json, id, storyboards) json.array do - storyboards.each do |storyboard| + storyboards.each do |sb| json.object do - json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard.width}&height=#{storyboard.height}" - json.field "templateUrl", storyboard.url - json.field "width", storyboard.width - json.field "height", storyboard.height - json.field "count", storyboard.count - json.field "interval", storyboard.interval - json.field "storyboardWidth", storyboard.storyboard_width - json.field "storyboardHeight", storyboard.storyboard_height - json.field "storyboardCount", storyboard.storyboard_count + json.field "url", "/api/v1/storyboards/#{id}?width=#{sb.width}&height=#{sb.height}" + json.field "templateUrl", sb.url.to_s + json.field "width", sb.width + json.field "height", sb.height + json.field "count", sb.count + json.field "interval", sb.interval + json.field "storyboardWidth", sb.columns + json.field "storyboardHeight", sb.rows + json.field "storyboardCount", sb.images_count end end end diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 78f91a2e..fb083934 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -187,15 +187,14 @@ module Invidious::Routes::API::V1::Videos haltf env, 500 end - storyboards = video.storyboards - width = env.params.query["width"]? - height = env.params.query["height"]? + width = env.params.query["width"]?.try &.to_i + height = env.params.query["height"]?.try &.to_i if !width && !height response = JSON.build do |json| json.object do json.field "storyboards" do - Invidious::JSONify::APIv1.storyboards(json, id, storyboards) + Invidious::JSONify::APIv1.storyboards(json, id, video.storyboards) end end end @@ -205,32 +204,37 @@ module Invidious::Routes::API::V1::Videos env.response.content_type = "text/vtt" - storyboard = storyboards.select { |sb| width == "#{sb.width}" || height == "#{sb.height}" } + # Select a storyboard matching the user's provided width/height + storyboard = video.storyboards.select { |x| x.width == width || x.height == height } + haltf env, 404 if storyboard.empty? - if storyboard.empty? - haltf env, 404 - else - storyboard = storyboard[0] - end + # Alias variable, to make the code below esaier to read + sb = storyboard[0] - WebVTT.build do |vtt| - start_time = 0.milliseconds - end_time = storyboard.interval.milliseconds + # Some base URL segments that we'll use to craft the final URLs + work_url = sb.proxied_url.dup + template_path = sb.proxied_url.path - storyboard.storyboard_count.times do |i| - url = storyboard.url - authority = /(i\d?).ytimg.com/.match!(url)[1]? + # Initialize cue timing variables + time_delta = sb.interval.milliseconds + start_time = 0.milliseconds + end_time = time_delta - 1.milliseconds - url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "") - url = "#{HOST_URL}/sb/#{authority}/#{url}" + # Build a VTT file for VideoJS-vtt plugin + return WebVTT.build do |vtt| + sb.images_count.times do |i| + # Replace the variable component part of the path + work_url.path = template_path.sub("$M", i) - storyboard.storyboard_height.times do |j| - storyboard.storyboard_width.times do |k| - current_cue_url = "#{url}#xywh=#{storyboard.width * k},#{storyboard.height * j},#{storyboard.width - 2},#{storyboard.height}" - vtt.cue(start_time, end_time, current_cue_url) + sb.rows.times do |j| + sb.columns.times do |k| + # The URL fragment represents the offset of the thumbnail inside the storyboard image + work_url.fragment = "xywh=#{sb.width * k},#{sb.height * j},#{sb.width - 2},#{sb.height}" - start_time += storyboard.interval.milliseconds - end_time += storyboard.interval.milliseconds + vtt.cue(start_time, end_time, work_url.to_s) + + start_time += time_delta + end_time += time_delta end end end diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr index 797fba12..f6df187f 100644 --- a/src/invidious/videos/storyboard.cr +++ b/src/invidious/videos/storyboard.cr @@ -3,74 +3,110 @@ require "http/params" module Invidious::Videos struct Storyboard - getter url : String + # Template URL + getter url : URI + getter proxied_url : URI + + # Thumbnail parameters getter width : Int32 getter height : Int32 getter count : Int32 getter interval : Int32 - getter storyboard_width : Int32 - getter storyboard_height : Int32 - getter storyboard_count : Int32 + + # Image (storyboard) parameters + getter rows : Int32 + getter columns : Int32 + getter images_count : Int32 def initialize( *, @url, @width, @height, @count, @interval, - @storyboard_width, @storyboard_height, @storyboard_count + @rows, @columns, @images_count ) + authority = /(i\d?).ytimg.com/.match(@url.host.not_nil!).not_nil![1]? + + @proxied_url = URI.parse(HOST_URL) + @proxied_url.path = "/sb/#{authority}#{@url.path}" + @proxied_url.query = @url.query end # Parse the JSON structure from Youtube - def self.from_yt_json(container : JSON::Any) + def self.from_yt_json(container : JSON::Any) : Array(Storyboard) + # Livestream storyboards are a bit different + # TODO: document exactly how + if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s + return [Storyboard.new( + url: URI.parse(storyboard.split("#")[0]), + width: 106, + height: 60, + count: -1, + interval: 5000, + rows: 3, + columns: 3, + images_count: -1 + )] + end + + # Split the storyboard string into chunks + # + # General format (whitespaces added for legibility): + # https://i.ytimg.com/sb//storyboard3_L$L/$N.jpg?sqp= + # | 48 # 27 # 100 # 10 # 10 # 0 # default # rs$ + # | 80 # 45 # 95 # 10 # 10 # 10000 # M$M # rs$ + # | 160 # 90 # 95 # 5 # 5 # 10000 # M$M # rs$ + # storyboards = container.dig?("playerStoryboardSpecRenderer", "spec") .try &.as_s.split("|") - if !storyboards - if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s - return [Storyboard.new( - url: storyboard.split("#")[0], - width: 106, - height: 60, - count: -1, - interval: 5000, - storyboard_width: 3, - storyboard_height: 3, - storyboard_count: -1, - )] - end - end - - items = [] of Storyboard - - return items if !storyboards + return [] of Storyboard if !storyboards + # The base URL is the first chunk url = URI.parse(storyboards.shift) - params = HTTP::Params.parse(url.query || "") + params = url.query_params - storyboards.each_with_index do |sb, i| - width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#") - params["sigh"] = sigh - url.query = params.to_s + return storyboards.map_with_index do |sb, i| + # Separate the different storyboard parameters: + # width/height: respective dimensions, in pixels, of a single thumbnail + # count: how many thumbnails are displayed across the full video + # columns/rows: maximum amount of thumbnails that can be stuffed in a + # single image, horizontally and vertically. + # interval: interval between two thumbnails, in milliseconds + # sigh: URL cryptographic signature + width, height, count, columns, rows, interval, _, sigh = sb.split("#") width = width.to_i height = height.to_i count = count.to_i interval = interval.to_i - storyboard_width = storyboard_width.to_i - storyboard_height = storyboard_height.to_i - storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i + columns = columns.to_i + rows = rows.to_i - items << Storyboard.new( - url: url.to_s.sub("$L", i).sub("$N", "M$M"), + # Add the signature to the URL + params["sigh"] = sigh + url.query = params.to_s + + # Replace the template parts with what we have + url.path = url.path.sub("$L", i).sub("$N", "M$M") + + # This value represents the maximum amount of thumbnails that can fit + # in a single image. The last image (or the only one for short videos) + # will contain less thumbnails than that. + thumbnails_per_image = columns * rows + + # This value represents the total amount of storyboards required to + # hold all of the thumbnails. It can't be less than 1. + images_count = (count / thumbnails_per_image).ceil.to_i + + Storyboard.new( + url: url, width: width, height: height, count: count, interval: interval, - storyboard_width: storyboard_width, - storyboard_height: storyboard_height, - storyboard_count: storyboard_count + rows: rows, + columns: columns, + images_count: images_count, ) end - - items end end end From 7b50388eafcd458221f3deec03bf5a0829244529 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 8 Oct 2023 20:36:32 +0200 Subject: [PATCH 13/30] Storyboards: Fix broken first storyboard --- src/invidious/videos.cr | 2 +- src/invidious/videos/storyboard.cr | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 73321909..28cbb311 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -178,7 +178,7 @@ struct Video def storyboards container = info.dig?("storyboards") || JSON::Any.new("{}") - return IV::Videos::Storyboard.from_yt_json(container) + return IV::Videos::Storyboard.from_yt_json(container, self.length_seconds) end def paid diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr index f6df187f..61aafe37 100644 --- a/src/invidious/videos/storyboard.cr +++ b/src/invidious/videos/storyboard.cr @@ -30,7 +30,7 @@ module Invidious::Videos end # Parse the JSON structure from Youtube - def self.from_yt_json(container : JSON::Any) : Array(Storyboard) + def self.from_yt_json(container : JSON::Any, length_seconds : Int32) : Array(Storyboard) # Livestream storyboards are a bit different # TODO: document exactly how if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s @@ -70,8 +70,9 @@ module Invidious::Videos # columns/rows: maximum amount of thumbnails that can be stuffed in a # single image, horizontally and vertically. # interval: interval between two thumbnails, in milliseconds + # name: storyboard filename. Usually "M$M" or "default" # sigh: URL cryptographic signature - width, height, count, columns, rows, interval, _, sigh = sb.split("#") + width, height, count, columns, rows, interval, name, sigh = sb.split("#") width = width.to_i height = height.to_i @@ -85,7 +86,7 @@ module Invidious::Videos url.query = params.to_s # Replace the template parts with what we have - url.path = url.path.sub("$L", i).sub("$N", "M$M") + url.path = url.path.sub("$L", i).sub("$N", name) # This value represents the maximum amount of thumbnails that can fit # in a single image. The last image (or the only one for short videos) @@ -96,6 +97,12 @@ module Invidious::Videos # hold all of the thumbnails. It can't be less than 1. images_count = (count / thumbnails_per_image).ceil.to_i + # Compute the interval when needed (in general, that's only required + # for the first "default" storyboard). + if interval == 0 + interval = ((length_seconds / count) * 1_000).to_i + end + Storyboard.new( url: url, width: width, From a335bc0814d3253852ed5b5cf58b75d9f7b6cd70 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 20 Oct 2023 23:37:12 +0200 Subject: [PATCH 14/30] Storyboards: Fix some small logic mistakes --- src/invidious/videos/storyboard.cr | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr index 61aafe37..35012663 100644 --- a/src/invidious/videos/storyboard.cr +++ b/src/invidious/videos/storyboard.cr @@ -25,7 +25,7 @@ module Invidious::Videos authority = /(i\d?).ytimg.com/.match(@url.host.not_nil!).not_nil![1]? @proxied_url = URI.parse(HOST_URL) - @proxied_url.path = "/sb/#{authority}#{@url.path}" + @proxied_url.path = "/sb/#{authority}/#{@url.path.lchop("/sb/")}" @proxied_url.query = @url.query end @@ -60,8 +60,7 @@ module Invidious::Videos return [] of Storyboard if !storyboards # The base URL is the first chunk - url = URI.parse(storyboards.shift) - params = url.query_params + base_url = URI.parse(storyboards.shift) return storyboards.map_with_index do |sb, i| # Separate the different storyboard parameters: @@ -81,9 +80,13 @@ module Invidious::Videos columns = columns.to_i rows = rows.to_i + # Copy base URL object, so that we can modify it + url = base_url.dup + # Add the signature to the URL + params = url.query_params params["sigh"] = sigh - url.query = params.to_s + url.query_params = params # Replace the template parts with what we have url.path = url.path.sub("$L", i).sub("$N", name) From 5b05f3bd147c6cf9421587565dea2b11640f1206 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 16 Aug 2024 11:28:35 +0200 Subject: [PATCH 15/30] Storyboards: Workarounds for videojs-vtt-thumbnails The workarounds are as follow: * Unescape HTML entities * Always use 0:00:00.000 for cue start/end --- src/invidious/routes/api/v1/videos.cr | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index fb083934..ab03df01 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -1,3 +1,5 @@ +require "html" + module Invidious::Routes::API::V1::Videos def self.videos(env) locale = env.get("preferences").as(Preferences).locale @@ -216,12 +218,14 @@ module Invidious::Routes::API::V1::Videos template_path = sb.proxied_url.path # Initialize cue timing variables + # NOTE: videojs-vtt-thumbnails gets lost when the start and end times are not 0:00:000.000 + # TODO: Use proper end time when videojs-vtt-thumbnails is fixed time_delta = sb.interval.milliseconds start_time = 0.milliseconds - end_time = time_delta - 1.milliseconds + end_time = 0.milliseconds # time_delta - 1.milliseconds # Build a VTT file for VideoJS-vtt plugin - return WebVTT.build do |vtt| + vtt_file = WebVTT.build do |vtt| sb.images_count.times do |i| # Replace the variable component part of the path work_url.path = template_path.sub("$M", i) @@ -233,12 +237,18 @@ module Invidious::Routes::API::V1::Videos vtt.cue(start_time, end_time, work_url.to_s) - start_time += time_delta - end_time += time_delta + # TODO: uncomment these when videojs-vtt-thumbnails is fixed + # start_time += time_delta + # end_time += time_delta end end end end + + # videojs-vtt-thumbnails is not compliant to the VTT specification, it + # doesn't unescape the HTML entities, so we have to do it here: + # TODO: remove this when we migrate to VideoJS 8 + return HTML.unescape(vtt_file) end def self.annotations(env) From b795bdf2a4a50fc899fde9dc7b42b845a4588bfc Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 16 Aug 2024 12:10:22 +0200 Subject: [PATCH 16/30] HTML: Sort playlists alphabetically in watch page drop down --- src/invidious/database/playlists.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/invidious/database/playlists.cr b/src/invidious/database/playlists.cr index c6754a1e..08aa719a 100644 --- a/src/invidious/database/playlists.cr +++ b/src/invidious/database/playlists.cr @@ -140,6 +140,7 @@ module Invidious::Database::Playlists request = <<-SQL SELECT id,title FROM playlists WHERE author = $1 AND id LIKE 'IV%' + ORDER BY title SQL PG_DB.query_all(request, email, as: {String, String}) From 764965c441a789e0be417648716f575067d9201e Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 17 Aug 2024 12:20:53 +0200 Subject: [PATCH 17/30] Storyboards: Fix lint error --- src/invidious/videos/storyboard.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr index 35012663..a72c2f55 100644 --- a/src/invidious/videos/storyboard.cr +++ b/src/invidious/videos/storyboard.cr @@ -22,7 +22,7 @@ module Invidious::Videos *, @url, @width, @height, @count, @interval, @rows, @columns, @images_count ) - authority = /(i\d?).ytimg.com/.match(@url.host.not_nil!).not_nil![1]? + authority = /(i\d?).ytimg.com/.match!(@url.host.not_nil!)[1]? @proxied_url = URI.parse(HOST_URL) @proxied_url.path = "/sb/#{authority}/#{@url.path.lchop("/sb/")}" From eb0f651812d7d01c038f5a052bf30fc8e26b877f Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 1 Oct 2023 19:39:53 +0200 Subject: [PATCH 18/30] Add a youtube URL sanitizer --- src/invidious/yt_backend/url_sanitizer.cr | 121 ++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 src/invidious/yt_backend/url_sanitizer.cr diff --git a/src/invidious/yt_backend/url_sanitizer.cr b/src/invidious/yt_backend/url_sanitizer.cr new file mode 100644 index 00000000..02bf77bf --- /dev/null +++ b/src/invidious/yt_backend/url_sanitizer.cr @@ -0,0 +1,121 @@ +require "uri" + +module UrlSanitizer + extend self + + ALLOWED_QUERY_PARAMS = { + channel: ["u", "user", "lb"], + playlist: ["list"], + search: ["q", "search_query", "sp"], + watch: [ + "v", # Video ID + "list", "index", # Playlist-related + "playlist", # Unnamed playlist (id,id,id,...) (embed-only?) + "t", "time_continue", "start", "end", # Timestamp + "lc", # Highlighted comment (watch page only) + ], + } + + # Returns wether the given string is an ASCII word. This is the same as + # running the following regex in US-ASCII locale: /^[\w-]+$/ + private def ascii_word?(str : String) : Bool + if str.bytesize == str.size + str.each_byte do |byte| + next if 'a'.ord <= byte <= 'z'.ord + next if 'A'.ord <= byte <= 'Z'.ord + next if '0'.ord <= byte <= '9'.ord + next if byte == '-'.ord || byte == '_'.ord + + return false + end + + return true + else + return false + end + end + + # Return which kind of parameters are allowed based on the + # first path component (breadcrumb 0). + private def determine_allowed(path_root : String) + case path_root + when "watch", "w", "v", "embed", "e", "shorts", "clip" + return :watch + when .starts_with?("@"), "c", "channel", "user", "profile", "attribution_link" + return :channel + when "playlist", "mix" + return :playlist + when "results", "search" + return :search + else # hashtag, post, trending, brand URLs, etc.. + return nil + end + end + + # Create a new URI::Param containing only the allowed parameters + private def copy_params(unsafe_params : URI::Params, allowed_type) : URI::Params + new_params = URI::Params.new + + ALLOWED_QUERY_PARAMS[allowed_type].each do |name| + if unsafe_params[name]? + # Only copy the last parameter, in case there is more than one + new_params[name] = unsafe_params.fetch_all(name)[-1] + end + end + + return new_params + end + + # Transform any user-supplied youtube URL into something we can trust + # and use across the code. + def process(str : String) : URI + # Because URI follows RFC3986 specifications, URL without a scheme + # will be parsed as a relative path. So we have to add a scheme ourselves. + str = "https://#{str}" if !str.starts_with?(/https?:\/\//) + + unsafe_uri = URI.parse(str) + new_uri = URI.new(path: "/") + + # Redirect to homepage for bogus URLs + return new_uri if (unsafe_uri.host.nil? || unsafe_uri.path.nil?) + + breadcrumbs = unsafe_uri.path + .split('/', remove_empty: true) + .compact_map do |bc| + # Exclude attempts at path trasversal + next if bc == "." || bc == ".." + + # Non-alnum characters are unlikely in a genuine URL + next if !ascii_word?(bc) + + bc + end + + # If nothing remains, it's either a legit URL to the homepage + # (who does that!?) or because we filtered some junk earlier. + return new_uri if breadcrumbs.empty? + + # Replace the original query parameters with the sanitized ones + case unsafe_uri.host.not_nil! + when .ends_with?("youtube.com") + # Use our sanitized path (not forgetting the leading '/') + new_uri.path = "/#{breadcrumbs.join('/')}" + + # Then determine which params are allowed, and copy them over + if allowed = determine_allowed(breadcrumbs[0]) + new_uri.query_params = copy_params(unsafe_uri.query_params, allowed) + end + when "youtu.be" + # Always redirect to the watch page + new_uri.path = "/watch" + + new_params = copy_params(unsafe_uri.query_params, :watch) + new_params["id"] = breadcrumbs[0] + + new_uri.query_params = new_params + end + + new_uri.host = nil # Safety measure + return new_uri + end +end From 4c0b5c314d68ea45e69de9673f0bf43bedf3acc4 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 5 Oct 2023 23:01:44 +0200 Subject: [PATCH 19/30] Search: Add support for youtu.be and youtube.com URLs --- src/invidious/routes/search.cr | 6 ++++++ src/invidious/search/query.cr | 27 +++++++++++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 5be33533..85aa1c7e 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -51,6 +51,12 @@ module Invidious::Routes::Search else user = env.get? "user" + # An URL was copy/pasted in the search box. + # Redirect the user to the appropriate page. + if query.is_url? + return env.redirect UrlSanitizer.process(query.text).to_s + end + begin items = query.process rescue ex : ChannelSearchException diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index e38845d9..f87c243e 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -48,11 +48,12 @@ module Invidious::Search ) # Get the raw search query string (common to all search types). In # Regular search mode, also look for the `search_query` URL parameter - if @type.regular? - @raw_query = params["q"]? || params["search_query"]? || "" - else - @raw_query = params["q"]? || "" - end + _raw_query = params["q"]? + _raw_query ||= params["search_query"]? if @type.regular? + _raw_query ||= "" + + # Remove surrounding whitespaces. Mostly useful for copy/pasted URLs. + @raw_query = _raw_query.strip # Get the page number (also common to all search types) @page = params["page"]?.try &.to_i? || 1 @@ -85,7 +86,7 @@ module Invidious::Search @filters = Filters.from_iv_params(params) @channel = params["channel"]? || "" - if @filters.default? && @raw_query.includes?(':') + if @filters.default? && @raw_query.index(/\w:\w/) # Parse legacy filters from query @filters, @channel, @query, subs = Filters.from_legacy_filters(@raw_query) else @@ -136,5 +137,19 @@ module Invidious::Search return params end + + # Checks if the query is a standalone URL + def is_url? : Bool + # Only supported in regular search mode + return false if !@type.regular? + + # If filters are present, that's a regular search + return false if !@filters.default? + + # Simple heuristics: domain name + return @raw_query.starts_with?( + /(https?:\/\/)?(www\.)?(m\.)?youtu(\.be|be\.com)\// + ) + end end end From 31a80420ec9f4dbd61a7145044f5e1797d4e0dd0 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 13 Feb 2024 21:46:12 +0100 Subject: [PATCH 20/30] Search: Add URL search inhibition logic --- src/invidious/search/query.cr | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index f87c243e..b3db0f63 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -20,6 +20,9 @@ module Invidious::Search property region : String? property channel : String = "" + # Flag that indicates if the smart search features have been disabled. + @inhibit_ssf : Bool = false + # Return true if @raw_query is either `nil` or empty private def empty_raw_query? return @raw_query.empty? @@ -55,6 +58,13 @@ module Invidious::Search # Remove surrounding whitespaces. Mostly useful for copy/pasted URLs. @raw_query = _raw_query.strip + # Check for smart features (ex: URL search) inhibitor (exclamation mark). + # If inhibitor is present, remove it. + if @raw_query.starts_with?('!') + @inhibit_ssf = true + @raw_query = @raw_query[1..] + end + # Get the page number (also common to all search types) @page = params["page"]?.try &.to_i? || 1 @@ -140,6 +150,9 @@ module Invidious::Search # Checks if the query is a standalone URL def is_url? : Bool + # If the smart features have been inhibited, don't go further. + return false if @inhibit_ssf + # Only supported in regular search mode return false if !@type.regular? From 78c5ba93c7f4eecf7aae623079c0c77f78670b67 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 17 Feb 2024 14:27:25 +0100 Subject: [PATCH 21/30] Misc: Clean some code in UrlSanitizer --- src/invidious/yt_backend/url_sanitizer.cr | 30 +++++++++++------------ 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/invidious/yt_backend/url_sanitizer.cr b/src/invidious/yt_backend/url_sanitizer.cr index 02bf77bf..725382ee 100644 --- a/src/invidious/yt_backend/url_sanitizer.cr +++ b/src/invidious/yt_backend/url_sanitizer.cr @@ -16,23 +16,21 @@ module UrlSanitizer ], } - # Returns wether the given string is an ASCII word. This is the same as + # Returns whether the given string is an ASCII word. This is the same as # running the following regex in US-ASCII locale: /^[\w-]+$/ private def ascii_word?(str : String) : Bool - if str.bytesize == str.size - str.each_byte do |byte| - next if 'a'.ord <= byte <= 'z'.ord - next if 'A'.ord <= byte <= 'Z'.ord - next if '0'.ord <= byte <= '9'.ord - next if byte == '-'.ord || byte == '_'.ord + return false if str.bytesize != str.size - return false - end + str.each_byte do |byte| + next if 'a'.ord <= byte <= 'z'.ord + next if 'A'.ord <= byte <= 'Z'.ord + next if '0'.ord <= byte <= '9'.ord + next if byte == '-'.ord || byte == '_'.ord - return true - else return false end + + return true end # Return which kind of parameters are allowed based on the @@ -74,12 +72,15 @@ module UrlSanitizer str = "https://#{str}" if !str.starts_with?(/https?:\/\//) unsafe_uri = URI.parse(str) + unsafe_host = unsafe_uri.host + unsafe_path = unsafe_uri.path + new_uri = URI.new(path: "/") # Redirect to homepage for bogus URLs - return new_uri if (unsafe_uri.host.nil? || unsafe_uri.path.nil?) + return new_uri if (unsafe_host.nil? || unsafe_path.nil?) - breadcrumbs = unsafe_uri.path + breadcrumbs = unsafe_path .split('/', remove_empty: true) .compact_map do |bc| # Exclude attempts at path trasversal @@ -96,7 +97,7 @@ module UrlSanitizer return new_uri if breadcrumbs.empty? # Replace the original query parameters with the sanitized ones - case unsafe_uri.host.not_nil! + case unsafe_host when .ends_with?("youtube.com") # Use our sanitized path (not forgetting the leading '/') new_uri.path = "/#{breadcrumbs.join('/')}" @@ -115,7 +116,6 @@ module UrlSanitizer new_uri.query_params = new_params end - new_uri.host = nil # Safety measure return new_uri end end From 85deea5aca4877507bb8850e5e3e168d968328ad Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 27 Apr 2024 23:21:27 +0200 Subject: [PATCH 22/30] Search: Change smart search inhibitor to a backslash --- src/invidious/search/query.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index b3db0f63..a93bb3f9 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -58,9 +58,9 @@ module Invidious::Search # Remove surrounding whitespaces. Mostly useful for copy/pasted URLs. @raw_query = _raw_query.strip - # Check for smart features (ex: URL search) inhibitor (exclamation mark). + # Check for smart features (ex: URL search) inhibitor (backslash). # If inhibitor is present, remove it. - if @raw_query.starts_with?('!') + if @raw_query.starts_with?('\\') @inhibit_ssf = true @raw_query = @raw_query[1..] end From c606465708720c953c37032624ff31e5e9d841ab Mon Sep 17 00:00:00 2001 From: Colin Leroy-Mira Date: Mon, 19 Aug 2024 09:34:51 +0200 Subject: [PATCH 23/30] Proxify formatStreams URLs too --- src/invidious/jsonify/api_v1/video_json.cr | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 59714828..e4379601 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -162,7 +162,13 @@ module Invidious::JSONify::APIv1 json.array do video.fmt_stream.each do |fmt| json.object do - json.field "url", fmt["url"] + if proxy + json.field "url", Invidious::HttpServer::Utils.proxy_video_url( + fmt["url"].to_s, absolute: true + ) + else + json.field "url", fmt["url"] + end json.field "itag", fmt["itag"].as_i.to_s json.field "type", fmt["mimeType"] json.field "quality", fmt["quality"] From 22b35c453ede48e36db1657c5b8e879f3cc70a56 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 25 Jul 2024 20:12:17 -0700 Subject: [PATCH 24/30] Ameba: Fix Style/WhileTrue --- src/invidious/routes/video_playback.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index ec18f3b8..24693662 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -131,7 +131,7 @@ module Invidious::Routes::VideoPlayback end # TODO: Record bytes written so we can restart after a chunk fails - while true + loop do if !range_end && content_length range_end = content_length end From f66068976e5f077d363769055b7533cd0f85d6d0 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 26 Jul 2024 19:19:31 -0700 Subject: [PATCH 25/30] Ameba: Fix Naming/PredicateName --- src/invidious/helpers/serialized_yt_data.cr | 4 ++-- src/invidious/jsonify/api_v1/video_json.cr | 2 +- src/invidious/user/imports.cr | 4 ++-- src/invidious/videos.cr | 20 ++++++++++++++++++-- src/invidious/views/watch.ecr | 2 +- 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 31a3cf44..463d5557 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -90,7 +90,7 @@ struct SearchVideo json.field "lengthSeconds", self.length_seconds json.field "liveNow", self.live_now json.field "premium", self.premium - json.field "isUpcoming", self.is_upcoming + json.field "isUpcoming", self.upcoming? if self.premiere_timestamp json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix @@ -109,7 +109,7 @@ struct SearchVideo to_json(nil, json) end - def is_upcoming + def upcoming? premiere_timestamp ? true : false end end diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 59714828..2d41ed3b 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -63,7 +63,7 @@ module Invidious::JSONify::APIv1 json.field "isListed", video.is_listed json.field "liveNow", video.live_now json.field "isPostLiveDvr", video.post_live_dvr - json.field "isUpcoming", video.is_upcoming + json.field "isUpcoming", video.upcoming? if video.premiere_timestamp json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index a70434ca..2b5f88f4 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -161,7 +161,7 @@ struct Invidious::User # Youtube # ------------------- - private def is_opml?(mimetype : String, extension : String) + private def opml?(mimetype : String, extension : String) opml_mimetypes = [ "application/xml", "text/xml", @@ -179,7 +179,7 @@ struct Invidious::User def from_youtube(user : User, body : String, filename : String, type : String) : Bool extension = filename.split(".").last - if is_opml?(type, extension) + if opml?(type, extension) subscriptions = XML.parse(body) user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel| channel["xmlUrl"].match!(/UC[a-zA-Z0-9_-]{22}/)[0] diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 6d0cf9ba..65b07fe8 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -280,7 +280,7 @@ struct Video info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil end - def is_vr : Bool? + def vr? : Bool? return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type end @@ -361,6 +361,21 @@ struct Video {% if flag?(:debug_macros) %} {{debug}} {% end %} end + # Macro to generate ? and = accessor methods for attributes in `info` + private macro predicate_bool(method_name, name) + # Return {{name.stringify}} from `info` + def {{method_name.id.underscore}}? : Bool + return info[{{name.stringify}}]?.try &.as_bool || false + end + + # Update {{name.stringify}} into `info` + def {{method_name.id.underscore}}=(value : Bool) + info[{{name.stringify}}] = JSON::Any.new(value) + end + + {% if flag?(:debug_macros) %} {{debug}} {% end %} + end + # Method definitions, using the macros above getset_string author @@ -382,11 +397,12 @@ struct Video getset_i64 likes getset_i64 views + # TODO: Make predicate_bool the default as to adhere to Crystal conventions getset_bool allowRatings getset_bool authorVerified getset_bool isFamilyFriendly getset_bool isListed - getset_bool isUpcoming + predicate_bool upcoming, isUpcoming end def get_video(id, refresh = true, region = nil, force_refresh = false) diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 36679bce..45c58a16 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -62,7 +62,7 @@ we're going to need to do it here in order to allow for translations. "params" => params, "preferences" => preferences, "premiere_timestamp" => video.premiere_timestamp.try &.to_unix, - "vr" => video.is_vr, + "vr" => video.vr?, "projection_type" => video.projection_type, "local_disabled" => CONFIG.disabled?("local"), "support_reddit" => true From d1cd7903882b23eedae6ff28441c1adc40b5be7b Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 26 Jul 2024 19:20:06 -0700 Subject: [PATCH 26/30] Ameba: Fix Lint/RedundantStringCoercion --- src/invidious/jsonify/api_v1/video_json.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 2d41ed3b..3625b8f1 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -109,7 +109,7 @@ module Invidious::JSONify::APIv1 # On livestreams, it's not present, so always fall back to the # current unix timestamp (up to mS precision) for compatibility. last_modified = fmt["lastModified"]? - last_modified ||= "#{Time.utc.to_unix_ms.to_s}000" + last_modified ||= "#{Time.utc.to_unix_ms}000" json.field "lmt", last_modified json.field "projectionType", fmt["projectionType"] From ecbea0b67b7b478597e40b530c0df8cd212e4faf Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 26 Jul 2024 19:22:42 -0700 Subject: [PATCH 27/30] Ameba: Fix Lint/ShadowingOuterLocalVar --- src/invidious/routes/api/v1/videos.cr | 4 ++-- src/invidious/user/imports.cr | 2 +- src/invidious/videos/transcript.cr | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 42282f44..c49a9b7b 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -116,7 +116,7 @@ module Invidious::Routes::API::V1::Videos else caption_xml = XML.parse(caption_xml) - webvtt = WebVTT.build(settings_field) do |webvtt| + webvtt = WebVTT.build(settings_field) do |builder| caption_nodes = caption_xml.xpath_nodes("//transcript/text") caption_nodes.each_with_index do |node, i| start_time = node["start"].to_f.seconds @@ -136,7 +136,7 @@ module Invidious::Routes::API::V1::Videos text = "#{md["text"]}" end - webvtt.cue(start_time, end_time, text) + builder.cue(start_time, end_time, text) end end end diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 2b5f88f4..533c18d9 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -115,7 +115,7 @@ struct Invidious::User playlists.each do |item| title = item["title"]?.try &.as_s?.try &.delete("<>") description = item["description"]?.try &.as_s?.try &.delete("\r") - privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy } + privacy = item["privacy"]?.try &.as_s?.try { |raw_pl_privacy_state| PlaylistPrivacy.parse? raw_pl_privacy_state } next if !title next if !description diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr index 9cd064c5..4bd9f820 100644 --- a/src/invidious/videos/transcript.cr +++ b/src/invidious/videos/transcript.cr @@ -110,13 +110,13 @@ module Invidious::Videos "Language" => @language_code, } - vtt = WebVTT.build(settings_field) do |vtt| + vtt = WebVTT.build(settings_field) do |builder| @lines.each do |line| # Section headers are excluded from the VTT conversion as to # match the regular captions returned from YouTube as much as possible next if line.is_a? HeadingLine - vtt.cue(line.start_ms, line.end_ms, line.line) + builder.cue(line.start_ms, line.end_ms, line.line) end end From b200ebfb6bc9de169d288c3d816332ea439fbdb6 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 21 Aug 2024 20:23:45 +0000 Subject: [PATCH 28/30] CSS: Remove extra space in default.css --- assets/css/default.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/css/default.css b/assets/css/default.css index 1445f65f..2cedcf0c 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -282,7 +282,7 @@ div.thumbnail > .bottom-right-overlay { display: flex; } -.searchbar .pure-form fieldset { +.searchbar .pure-form fieldset { padding: 0; flex: 1; } From 21ab5dc6680da3df62feed14c00104754f2479a4 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 22 Aug 2024 00:29:15 +0200 Subject: [PATCH 29/30] Storyboard: Revert cue timing "fix" --- src/invidious/routes/api/v1/videos.cr | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index ab03df01..c077b85e 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -218,11 +218,11 @@ module Invidious::Routes::API::V1::Videos template_path = sb.proxied_url.path # Initialize cue timing variables - # NOTE: videojs-vtt-thumbnails gets lost when the start and end times are not 0:00:000.000 - # TODO: Use proper end time when videojs-vtt-thumbnails is fixed + # NOTE: videojs-vtt-thumbnails gets lost when the cue times don't overlap + # (i.e: if cue[n] end time is 1:06:25.000, cue[n+1] start time should be 1:06:25.000) time_delta = sb.interval.milliseconds start_time = 0.milliseconds - end_time = 0.milliseconds # time_delta - 1.milliseconds + end_time = time_delta # Build a VTT file for VideoJS-vtt plugin vtt_file = WebVTT.build do |vtt| @@ -237,9 +237,8 @@ module Invidious::Routes::API::V1::Videos vtt.cue(start_time, end_time, work_url.to_s) - # TODO: uncomment these when videojs-vtt-thumbnails is fixed - # start_time += time_delta - # end_time += time_delta + start_time += time_delta + end_time += time_delta end end end From ccecc6d318ea80b2af3bf379b33700dcb6e16c97 Mon Sep 17 00:00:00 2001 From: syeopite <70992037+syeopite@users.noreply.github.com> Date: Sat, 24 Aug 2024 18:11:11 +0000 Subject: [PATCH 30/30] Fix lint errors introduced in #4146 and #4295 (#4876) * Ameba: Fix Naming/VariableNames Introduced in #4295 * Ameba: Fix Naming/PredicateName Introduced in #4146 --- src/invidious/channels/about.cr | 6 +++--- src/invidious/routes/search.cr | 2 +- src/invidious/search/query.cr | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 1380044a..13909527 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -50,12 +50,12 @@ def get_about_info(ucid, locale) : AboutChannel total_views = 0_i64 joined = Time.unix(0) - if ageGate = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer") + if age_gate_renderer = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer") description_node = nil - author = ageGate["channelTitle"].as_s + author = age_gate_renderer["channelTitle"].as_s ucid = initdata.dig("responseContext", "serviceTrackingParams", 0, "params", 0, "value").as_s author_url = "https://www.youtube.com/channel/#{ucid}" - author_thumbnail = ageGate.dig("avatar", "thumbnails", 0, "url").as_s + author_thumbnail = age_gate_renderer.dig("avatar", "thumbnails", 0, "url").as_s banner = nil is_family_friendly = false is_age_gated = true diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 85aa1c7e..44970922 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -53,7 +53,7 @@ module Invidious::Routes::Search # An URL was copy/pasted in the search box. # Redirect the user to the appropriate page. - if query.is_url? + if query.url? return env.redirect UrlSanitizer.process(query.text).to_s end diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr index a93bb3f9..c8e8cf7f 100644 --- a/src/invidious/search/query.cr +++ b/src/invidious/search/query.cr @@ -149,7 +149,7 @@ module Invidious::Search end # Checks if the query is a standalone URL - def is_url? : Bool + def url? : Bool # If the smart features have been inhibited, don't go further. return false if @inhibit_ssf