From b2a83991d16cc9fa65f71309cd4a745f005cdf61 Mon Sep 17 00:00:00 2001 From: absidue <48293849+absidue@users.noreply.github.com> Date: Fri, 20 Sep 2024 18:46:00 +0200 Subject: [PATCH 01/15] Fix parsing live_now and premiere_timestamp --- src/invidious/videos/parser.cr | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 811a0a03..c28ba634 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -224,8 +224,17 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp") .try { |t| Time.parse_rfc3339(t.as_s) } + premiere_timestamp ||= player_response.dig?( + "playabilityStatus", "liveStreamability", + "liveStreamabilityRenderer", "offlineSlate", + "liveStreamOfflineSlateRenderer", "scheduledStartTime" + ) + .try &.as_s.to_i64 + .try { |t| Time.unix(t) } + live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow") - .try &.as_bool || false + .try &.as_bool + live_now ||= video_details.dig?("isLive").try &.as_bool || false post_live_dvr = video_details.dig?("isPostLiveDvr") .try &.as_bool || false From cdf93b29e6376ea0c023da825aeb9d83ec588873 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 31 Oct 2024 11:51:33 +0100 Subject: [PATCH 02/15] Routing: Remove deprecated /api/v1/channels/.../:ucid routes --- src/invidious/routing.cr | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index ba05da19..9f76f15f 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -243,17 +243,16 @@ module Invidious::Routing # Channels get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home + get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest + get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases - + get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists + get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels - - {% for route in {"videos", "latest", "playlists", "community", "search"} %} - get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}} - get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}} - {% end %} + get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search # Posts get "/api/v1/post/:id", {{namespace}}::Channels, :post From 6da18ddc41a20cad06c736a28aef6064433e3bd5 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 31 Oct 2024 11:52:09 +0100 Subject: [PATCH 03/15] Routing: Also remove outdated comment about notification routes --- src/invidious/routing.cr | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 9f76f15f..9009062f 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -270,11 +270,6 @@ module Invidious::Routing # Authenticated - # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr - # - # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications - get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences From 09ccea1d31366e5f6ce1c0b3e13f3a8d84428184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Fri, 8 Nov 2024 22:01:23 +0100 Subject: [PATCH 04/15] remove usage of TVHTML5_SIMPLY_EMBEDDED_PLAYER --- src/invidious/videos/parser.cr | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index fb8935d9..8dab5881 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -124,14 +124,6 @@ def extract_video_info(video_id : String) new_player_response = try_fetch_streaming_data(video_id, client_config) end - # Last hope - # Only trigger if reason found or didn't work wth Android client. - # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token. - if reason && CONFIG.po_token.nil? - client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed - new_player_response = try_fetch_streaming_data(video_id, client_config) - end - # Replace player response and reset reason if !new_player_response.nil? # Preserve captions & storyboard data before replacement From 9d54cf903e4674577b096a47498ef42a06667bad Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Nov 2024 15:51:54 -0800 Subject: [PATCH 05/15] Update shard.yml metadata --- shard.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/shard.yml b/shard.yml index 513e5db3..e6a10c38 100644 --- a/shard.yml +++ b/shard.yml @@ -1,13 +1,12 @@ name: invidious -version: 0.20.1 +version: 2.20240825.2-dev authors: - Omar Roth - - Invidious team + - Invidious team -targets: - invidious: - main: src/invidious.cr +description: | + Invidious is an alternative front-end to YouTube dependencies: pg: @@ -40,6 +39,10 @@ development_dependencies: github: crystal-ameba/ameba version: ~> 1.6.1 -crystal: ">= 1.0.0, < 2.0.0" +crystal: ">= 1.10.0, < 2.0.0" license: AGPLv3 + +repository: https://github.com/iv-org/invidious +homepage: https://invidious.io +documentation: https://docs.invidious.io From 1a49e798c804e7841574838409dd0500010cbb16 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sat, 9 Nov 2024 21:52:06 +0100 Subject: [PATCH 06/15] Docker: Install tzdata in Dockerfile --- docker/Dockerfile | 6 +++--- docker/Dockerfile.arm64 | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3d9323fd..900c9e74 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM crystallang/crystal:1.12.1-alpine AS builder +FROM crystallang/crystal:1.12.2-alpine AS builder RUN apk add --no-cache sqlite-static yaml-static @@ -32,8 +32,8 @@ RUN if [[ "${release}" == 1 ]] ; then \ --link-flags "-lxml2 -llzma"; \ fi -FROM alpine:3.18 -RUN apk add --no-cache rsvg-convert ttf-opensans tini +FROM alpine:3.20 +RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ adduser -u 1000 -S invidious -G invidious diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index f054b326..ce9bab08 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -1,5 +1,6 @@ -FROM alpine:3.19 AS builder -RUN apk add --no-cache 'crystal=1.10.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-static zlib-static openssl-libs-static openssl-dev musl-dev xz-static +FROM alpine:3.20 AS builder +RUN apk add --no-cache 'crystal=1.12.2-r0' shards sqlite-static yaml-static yaml-dev libxml2-static \ + zlib-static openssl-libs-static openssl-dev musl-dev xz-static ARG release @@ -32,8 +33,8 @@ RUN if [[ "${release}" == 1 ]] ; then \ --link-flags "-lxml2 -llzma"; \ fi -FROM alpine:3.18 -RUN apk add --no-cache rsvg-convert ttf-opensans tini +FROM alpine:3.20 +RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ adduser -u 1000 -S invidious -G invidious From 8bf7e029782c86c3a1b7b4be1f7c6f3a9659d2eb Mon Sep 17 00:00:00 2001 From: syeopite <70992037+syeopite@users.noreply.github.com> Date: Sat, 9 Nov 2024 13:04:10 -0800 Subject: [PATCH 07/15] Change authors section to reflect current state --- shard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shard.yml b/shard.yml index e6a10c38..4c3f7526 100644 --- a/shard.yml +++ b/shard.yml @@ -2,8 +2,8 @@ name: invidious version: 2.20240825.2-dev authors: - - Omar Roth - Invidious team + - Contributors! description: | Invidious is an alternative front-end to YouTube From b9ad9bd72331e8a568bd11813cb0169bd9c2a831 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Fri, 8 Nov 2024 21:47:52 +0100 Subject: [PATCH 08/15] use WEB when po_token + android test suite when no po_token --- src/invidious/videos/parser.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index fb8935d9..b2744120 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -53,9 +53,9 @@ end def extract_video_info(video_id : String) # Init client config for the API client_config = YoutubeAPI::ClientConfig.new - # Use the WEB_CREATOR when po_token is configured because it fully only works on this client + # Use the WEB when po_token is configured if CONFIG.po_token - client_config.client_type = YoutubeAPI::ClientType::WebCreator + client_config.client_type = YoutubeAPI::ClientType::Web end # Fetch data from the player endpoint @@ -113,8 +113,8 @@ def extract_video_info(video_id : String) new_player_response = try_fetch_streaming_data(video_id, client_config) end - # Don't use Android client if po_token is passed because po_token doesn't - # work for Android client. + # Don't use Android test suite client if po_token is passed because po_token doesn't + # work for Android test suite client. if reason.nil? && CONFIG.po_token.nil? # Fetch the video streams using an Android client in order to get the # decrypted URLs and maybe fix throttling issues (#2194). See the From 82b1506cccc85ed4f3979d674a2754c35e0194fc Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Fri, 8 Nov 2024 21:56:24 +0100 Subject: [PATCH 09/15] remove usage of WebEmbeddedPlayer --- src/invidious/videos/parser.cr | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index b2744120..1f229df0 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -106,13 +106,6 @@ def extract_video_info(video_id : String) new_player_response = nil - # Second try in case WEB_CREATOR doesn't work with po_token. - # Only trigger if reason found and po_token configured. - if reason && CONFIG.po_token - client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer - new_player_response = try_fetch_streaming_data(video_id, client_config) - end - # Don't use Android test suite client if po_token is passed because po_token doesn't # work for Android test suite client. if reason.nil? && CONFIG.po_token.nil? From f3e93ca83d21ab7e766d931c3985cf291e96ad3e Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Fri, 8 Nov 2024 21:56:48 +0100 Subject: [PATCH 10/15] revert back to www.youtube.com when client_config.screen embed --- src/invidious/yt_backend/youtube_api.cr | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index e0a3181f..8f5aa61d 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -300,9 +300,8 @@ module YoutubeAPI end if client_config.screen == "EMBED" - # embedUrl https://www.google.com allow loading almost all video that are configured not embeddable client_context["thirdParty"] = { - "embedUrl" => "https://www.google.com/", + "embedUrl" => "https://www.youtube.com/embed/#{video_id}", } of String => String | Int64 end From 0f8f32bca8739dfc05edc70ac634bfebe763d927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20=28perso=29?= <4016501+unixfox@users.noreply.github.com> Date: Fri, 8 Nov 2024 23:33:19 +0100 Subject: [PATCH 11/15] remove explicit usage of WEB --- src/invidious/videos/parser.cr | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 1f229df0..65ba8a3d 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -53,10 +53,6 @@ end def extract_video_info(video_id : String) # Init client config for the API client_config = YoutubeAPI::ClientConfig.new - # Use the WEB when po_token is configured - if CONFIG.po_token - client_config.client_type = YoutubeAPI::ClientType::Web - end # Fetch data from the player endpoint player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config) From d2123b46829e57515e281ffd98b75dac3de6f379 Mon Sep 17 00:00:00 2001 From: Brahim Hadriche Date: Sat, 9 Nov 2024 17:49:06 -0500 Subject: [PATCH 12/15] Sort channel shorts API --- src/invidious/routes/api/v1/channels.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 2da76134..588bbc2a 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -197,6 +197,7 @@ module Invidious::Routes::API::V1::Channels get_channel() # Retrieve continuation from URL parameters + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" continuation = env.params.query["continuation"]? if channel.is_age_gated @@ -211,7 +212,7 @@ module Invidious::Routes::API::V1::Channels else begin videos, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) rescue ex return error_json(500, ex) From 2150264d849771df8f15bab172ab6d87eeb80c55 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 10 Nov 2024 18:00:26 +0100 Subject: [PATCH 13/15] Update CHANGELOG.md --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 061f977c..9cc36f12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ ### Full list of pull requests merged since the last release (newest first) +* API: Add "sort_by" parameter to channels/shorts endpoint ([#5071], thanks @iBicha) +* Docker: Install tzdata in Dockerfile ([#5070], by @SamantazFox) +* Videos: Stop using TVHTML5_SIMPLY_EMBEDDED_PLAYER ([#5063], thanks @unixfox) +* Routing: Deprecate old channel API routes ([#5045], by @SamantazFox) +* Videos: use WEB client instead of WEB CREATOR ([#4984], thanks @unixfox) +* Parsers: Fix parsing live_now and premiere_timestamp ([#4934], thanks @absidue) * Stale bot updates ([#5060], thanks @syeopite) * Channels: Fix "Youtube API returned error 400" ([#5059], by @SamantazFox) * Channels: Fix for live videos ([#5027], thanks @iBicha) @@ -52,15 +58,21 @@ [#4928]: https://github.com/iv-org/invidious/pull/4928 [#4930]: https://github.com/iv-org/invidious/pull/4930 [#4931]: https://github.com/iv-org/invidious/pull/4931 +[#4934]: https://github.com/iv-org/invidious/pull/4934 [#4942]: https://github.com/iv-org/invidious/pull/4942 +[#4984]: https://github.com/iv-org/invidious/pull/4984 [#4991]: https://github.com/iv-org/invidious/pull/4991 [#4993]: https://github.com/iv-org/invidious/pull/4993 [#4995]: https://github.com/iv-org/invidious/pull/4995 [#5027]: https://github.com/iv-org/invidious/pull/5027 [#5034]: https://github.com/iv-org/invidious/pull/5034 +[#5045]: https://github.com/iv-org/invidious/pull/5045 [#5046]: https://github.com/iv-org/invidious/pull/5046 [#5059]: https://github.com/iv-org/invidious/pull/5059 [#5060]: https://github.com/iv-org/invidious/pull/5060 +[#5063]: https://github.com/iv-org/invidious/pull/5063 +[#5070]: https://github.com/iv-org/invidious/pull/5070 +[#5071]: https://github.com/iv-org/invidious/pull/5071 ## v2.20240825.2 (2024-08-26) From 5d2dd40bc3e043caac7d85e7ddee3d2c5d40ccbb Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 10 Nov 2024 21:04:37 +0100 Subject: [PATCH 14/15] Release v2.20241110.0 --- CHANGELOG.md | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++- shard.yml | 2 +- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cc36f12..d8f3299c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,73 @@ # CHANGELOG -## vX.Y.0 (future) +## v2.20241110.0 + +### Wrap-up + +This release is most importantly here to fix to the annoying "Youtube API returned error 400" +error that prevented all channel pages from loading. + +If you're updating from the previous release, it provides no improvements on the ability to play +videos. If updating from a commit in-between release, it removes the "Please sign in" error caused +by a previous attempt at restoring video playback on large instances. + +In the preferences, a new option allows for control of video preload. When enabled, this option +tells the browser to load the video as soon as the page is loaded (this used to be the default). +When disabled, the video starts loading only when the "play" button is pressed. + +New interface languages available: Bulgarian, Welsh and Lombard + +New dependency required: `tzdata`. + +An HTTP proxy can be configured directly in Invidious, if needed. \ +**NOTE:** In that case, it is recommended to comment out `force_resolve`. + + +### New features & important changes + +#### For users + +* Channels: Fix "Youtube API returned error 400" error preventing channel pages from loading +* Channels: Shorts can now be sorted by "newest", "oldest" and "popular" +* Preferences: Addition of the new "preload" option +* New interface languages available: Bulgarian, Welsh and Lombard +* Added "Filipino (auto-generated)" to the list of caption languages available +* Lots of new translations from Weblate + +#### For instance owners + +* Allow the configuration of an HTTP proxy to talk to Youtube +* Invidious tries to reconnect to `inv_sig_helper` if the socket is closed +* The instance list is downloaded in the background to improve redirection speed +* New `colorize_logs` option makes each log level a different color + +#### For developpers + +* `/api/v1/channels/{id}/shorts` now supports the `sort-by` parameter with the following values: + `newest`, `oldest` and `popular` +* Older `/api/v1/channels/xyz/{id}` (tab name before UCID) were removed +* API/Search: New video metadata available: `isNew`, `is4k`, `is8k`, `isVr180`, `isVr360`, + `is3d` and `hasCaptions` + +### Bugs fixed + +#### User-side + +* Channels: The second page of shorts now loads as expected +* Channels: Fixed intermittent empty "playlists" tab +* Search: Fixed `youtu.be` URLs not being properly redirected to the watch page +* Fixed `DB::MappingException` error on the subscriptions feed (due to missing `tzdata` in docker) +* Switching to another instance is much faster +* Fixed an "invalid byte sequence" error when subscribing to a playlist +* Videos: Playback URLs were sometimes broken when cached and `inv_sig_helper` was used + +#### For instance owners + +* Fix `force_resolve` being ignored in some cases + +#### API + +* API/Videos: Fixed `live_now` and `premiere_timestamp` sometimes not having the right values ### Full list of pull requests merged since the last release (newest first) diff --git a/shard.yml b/shard.yml index 4c3f7526..2e4df045 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: invidious -version: 2.20240825.2-dev +version: 2.20241110.0 authors: - Invidious team From 98926047586154269bb269d01e3e52e60e044035 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 10 Nov 2024 21:40:32 +0100 Subject: [PATCH 15/15] Prepare for next release --- CHANGELOG.md | 3 +++ shard.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f3299c..5af38003 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # CHANGELOG +## vX.Y.0 (future) + + ## v2.20241110.0 ### Wrap-up diff --git a/shard.yml b/shard.yml index 2e4df045..af7e4186 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: invidious -version: 2.20241110.0 +version: 2.20241110.0-dev authors: - Invidious team