Merge remote-tracking branch 'upstream/master'

Reverting this commit before merging https://github.com/iv-org/invidious/pull/5149 from upstream

This reverts commit c61b2963ac.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -10,8 +10,10 @@ assignees: ''
 <!--
   BEFORE TRYING TO REPORT A BUG:
  
-  * Read the FAQ!
+  * Read the FAQ: https://docs.invidious.io/faq/!
-  * Use the search function to check if there is already an issue open for your problem!
+  * Use the search function to check if there is already an issue open for your problem: https://github.com/search?q=repo%3Aiv-org%2Finvidious+replace+me+with+your+bug&type=issues!
+
+  MAKE SURE TO FOLLOW THE TWO STEPS ABOVE BEFORE REPORTING A BUG. A BUG THAT ALREADY EXIST WILL IMMEDIATELY CLOSED.
 
   If you want to suggest a new feature please use "Feature request" instead
   If you want to suggest an enhancement to an existing feature please use "Enhancement" instead

.github/workflows/build-nightly-container.yml

@@ -23,19 +23,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Install Crystal
-        uses: crystal-lang/install-crystal@v1.8.2
-        with:
-          crystal: 1.12.2
-
-      - name: Run lint
-        run: |
-          if ! crystal tool format --check; then
-            crystal tool format
-            git diff
-            exit 1
-          fi
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
         with:

.github/workflows/build-stable-container.yml

@@ -14,19 +14,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Install Crystal
-        uses: crystal-lang/install-crystal@v1.8.2
-        with:
-          crystal: 1.12.2
-
-      - name: Run lint
-        run: |
-          if ! crystal tool format --check; then
-            crystal tool format
-            git diff
-            exit 1
-          fi
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
         with:

.github/workflows/ci.yml

@@ -38,11 +38,10 @@ jobs:
       matrix:
         stable: [true]
         crystal:
-          - 1.10.1
-          - 1.11.2
           - 1.12.1
           - 1.13.2
           - 1.14.0
+          - 1.15.0
         include:
           - crystal: nightly
             stable: false
@@ -65,7 +64,9 @@ jobs:
       - name: Cache Shards
         uses: actions/cache@v3
         with:
-          path: ./lib
+          path: |
+            ./lib
+            ./bin
           key: shards-${{ hashFiles('shard.lock') }}
 
       - name: Install Shards
@@ -77,14 +78,6 @@ jobs:
       - name: Run tests
         run: crystal spec
 
-      - name: Run lint
-        run: |
-          if ! crystal tool format --check; then
-            crystal tool format
-            git diff
-            exit 1
-          fi
-
       - name: Build
         run: crystal build --warnings all --error-on-warnings --error-trace src/invidious.cr
 
@@ -130,14 +123,19 @@ jobs:
       - name: Test Docker
         run: while curl -Isf http://localhost:3000; do sleep 1; done
 
-  ameba_lint:
+  lint:
+
     runs-on: ubuntu-latest
+
+    continue-on-error: true
+
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true
 
       - name: Install Crystal
+        id: lint_step_install_crystal
         uses: crystal-lang/install-crystal@v1.8.0
         with:
           crystal: latest
@@ -148,10 +146,21 @@ jobs:
           path: |
             ./lib
             ./bin
-          key: shards-${{ hashFiles('shard.lock') }}
+          key: shards-${{ hashFiles('shard.lock') }}-${{ steps.lint_step_install_crystal.outputs.crystal }}
 
       - name: Install Shards
-        run: shards install
+        run: |
+          if ! shards check; then
+            shards install
+          fi
+
+      - name: Check Crystal formatter compliance
+        run: |
+          if ! crystal tool format --check; then
+            crystal tool format
+            git diff
+            exit 1
+          fi
 
       - name: Run Ameba linter
         run: bin/ameba

.github/workflows/stale.yml

@@ -13,14 +13,11 @@ jobs:
     - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        days-before-stale: 365
+        days-before-stale: 730
-        days-before-pr-stale: 90 
+        days-before-pr-stale: -1 
-        days-before-close: 30
+        days-before-close: 60
-        exempt-pr-labels: blocked,exempt-stale
         stale-issue-message: 'This issue has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely outdated. If you think this issue is still relevant and applicable, you just have to post a comment and it will be unmarked.'
-        stale-pr-message: 'This pull request has been automatically marked as stale and will be closed in 30 days because it has not had recent activity and is much likely abandoned or outdated. If you think this pull request is still relevant and applicable, you just have to post a comment and it will be unmarked.'
         stale-issue-label: "stale"
-        stale-pr-label: "stale"
         ascending: true
-        # Never mark feature requests/enhancements as stale
+        # Exempt the following types of issues from being staled
-        exempt-issue-labels: "feature-request,enhancement,exempt-stale"
+        exempt-issue-labels: "feature-request,enhancement,discussion,exempt-stale"

CHANGELOG.md

@@ -3,8 +3,92 @@
 ## vX.Y.0 (future)
 
 
+## v2.20241110.0
+
+### Wrap-up
+
+This release is most importantly here to fix to the annoying "Youtube API returned error 400"
+error that prevented all channel pages from loading.
+
+If you're updating from the previous release, it provides no improvements on the ability to play
+videos. If updating from a commit in-between release, it removes the "Please sign in" error caused
+by a previous attempt at restoring video playback on large instances.
+
+In the preferences, a new option allows for control of video preload. When enabled, this option
+tells the browser to load the video as soon as the page is loaded (this used to be the default).
+When disabled, the video starts loading only when the "play" button is pressed.
+
+New interface languages available: Bulgarian, Welsh and Lombard
+
+New dependency required: `tzdata`.
+
+An HTTP proxy can be configured directly in Invidious, if needed. \
+**NOTE:** In that case, it is recommended to comment out `force_resolve`.
+
+
+### New features & important changes
+
+#### For users
+
+* Channels: Fix "Youtube API returned error 400" error preventing channel pages from loading
+* Channels: Shorts can now be sorted by "newest", "oldest" and "popular"
+* Preferences: Addition of the new "preload" option
+* New interface languages available: Bulgarian, Welsh and Lombard
+* Added "Filipino (auto-generated)" to the list of caption languages available
+* Lots of new translations from Weblate
+
+#### For instance owners
+
+* Allow the configuration of an HTTP proxy to talk to Youtube
+* Invidious tries to reconnect to `inv_sig_helper` if the socket is closed
+* The instance list is downloaded in the background to improve redirection speed
+* New `colorize_logs` option makes each log level a different color
+
+#### For developpers
+
+* `/api/v1/channels/{id}/shorts` now supports the `sort-by` parameter with the following values:
+  `newest`, `oldest` and `popular`
+* Older `/api/v1/channels/xyz/{id}` (tab name before UCID) were removed
+* API/Search: New video metadata available: `isNew`, `is4k`, `is8k`, `isVr180`, `isVr360`,
+  `is3d` and `hasCaptions`
+
+### Bugs fixed
+
+#### User-side
+
+* Channels: The second page of shorts now loads as expected
+* Channels: Fixed intermittent empty "playlists" tab
+* Search: Fixed `youtu.be` URLs not being properly redirected to the watch page
+* Fixed `DB::MappingException` error on the subscriptions feed (due to missing `tzdata` in docker)
+* Switching to another instance is much faster
+* Fixed an "invalid byte sequence" error when subscribing to a playlist
+* Videos: Playback URLs were sometimes broken when cached and `inv_sig_helper` was used
+
+#### For instance owners
+
+* Fix `force_resolve` being ignored in some cases
+
+#### API
+
+* API/Videos: Fixed `live_now` and `premiere_timestamp` sometimes not having the right values
+
+
 ### Full list of pull requests merged since the last release (newest first)
 
+* API: Add "sort_by" parameter to channels/shorts endpoint ([#5071], thanks @iBicha)
+* Docker: Install tzdata in Dockerfile ([#5070], by @SamantazFox)
+* Videos: Stop using TVHTML5_SIMPLY_EMBEDDED_PLAYER ([#5063], thanks @unixfox)
+* Routing: Deprecate old channel API routes ([#5045], by @SamantazFox)
+* Videos: use WEB client instead of WEB CREATOR ([#4984], thanks @unixfox)
+* Parsers: Fix parsing live_now and premiere_timestamp ([#4934], thanks @absidue)
+* Stale bot updates ([#5060], thanks @syeopite)
+* Channels: Fix "Youtube API returned error 400" ([#5059], by @SamantazFox)
+* Channels: Fix for live videos ([#5027], thanks @iBicha)
+* Locales: Add Bulgarian, Welsh and Lombard to the list ([#5046], by @SamantazFox)
+* Shards: Update database dependencies ([#5034], by @SamantazFox)
+* Logger: Add color support for different log levels ([#4931], thanks @Fijxu)
+* Fix named arg syntax when passing force_resolve ([#4754], thanks @syeopite)
+* Use make_client instead of calling HTTP::Client ([#4709], thanks @syeopite)
 * Add "Filipino (auto-generated)" to the list of caption languages ([#4995], by @SamantazFox)
 * Makefile: Add MT option to enable the 'preview_mt' flag ([#4993], by @SamantazFox)
 * SigHelper: Reconnect to signature helper ([#4991], thanks @Fijxu)
@@ -31,7 +115,9 @@
 [#4270]: https://github.com/iv-org/invidious/pull/4270
 [#4326]: https://github.com/iv-org/invidious/pull/4326
 [#4652]: https://github.com/iv-org/invidious/pull/4652
+[#4709]: https://github.com/iv-org/invidious/pull/4709
 [#4750]: https://github.com/iv-org/invidious/pull/4750
+[#4754]: https://github.com/iv-org/invidious/pull/4754
 [#4850]: https://github.com/iv-org/invidious/pull/4850
 [#4862]: https://github.com/iv-org/invidious/pull/4862
 [#4863]: https://github.com/iv-org/invidious/pull/4863
@@ -41,10 +127,22 @@
 [#4923]: https://github.com/iv-org/invidious/pull/4923
 [#4928]: https://github.com/iv-org/invidious/pull/4928
 [#4930]: https://github.com/iv-org/invidious/pull/4930
+[#4931]: https://github.com/iv-org/invidious/pull/4931
+[#4934]: https://github.com/iv-org/invidious/pull/4934
 [#4942]: https://github.com/iv-org/invidious/pull/4942
+[#4984]: https://github.com/iv-org/invidious/pull/4984
 [#4991]: https://github.com/iv-org/invidious/pull/4991
 [#4993]: https://github.com/iv-org/invidious/pull/4993
 [#4995]: https://github.com/iv-org/invidious/pull/4995
+[#5027]: https://github.com/iv-org/invidious/pull/5027
+[#5034]: https://github.com/iv-org/invidious/pull/5034
+[#5045]: https://github.com/iv-org/invidious/pull/5045
+[#5046]: https://github.com/iv-org/invidious/pull/5046
+[#5059]: https://github.com/iv-org/invidious/pull/5059
+[#5060]: https://github.com/iv-org/invidious/pull/5060
+[#5063]: https://github.com/iv-org/invidious/pull/5063
+[#5070]: https://github.com/iv-org/invidious/pull/5070
+[#5071]: https://github.com/iv-org/invidious/pull/5071
 
 
 ## v2.20240825.2 (2024-08-26)

assets/js/handlers.js

@@ -91,7 +91,7 @@
         var count = document.getElementById('count');
         count.textContent--;
 
-        var url = '/token_ajax?action_revoke_token=1&redirect=false' +
+        var url = '/token_ajax?action=revoke_token&redirect=false' +
             '&referer=' + encodeURIComponent(location.href) +
             '&session=' + target.getAttribute('data-session');
 
@@ -111,7 +111,7 @@
         var count = document.getElementById('count');
         count.textContent--;
 
-        var url = '/subscription_ajax?action_remove_subscriptions=1&redirect=false' +
+        var url = '/subscription_ajax?action=remove_subscriptions&redirect=false' +
             '&referer=' + encodeURIComponent(location.href) +
             '&c=' + target.getAttribute('data-ucid');
 

assets/js/playlist_widget.js

@@ -6,7 +6,7 @@ function add_playlist_video(target) {
     var select = target.parentNode.children[0].children[1];
     var option = select.children[select.selectedIndex];
 
-    var url = '/playlist_ajax?action_add_video=1&redirect=false' +
+    var url = '/playlist_ajax?action=add_video&redirect=false' +
         '&video_id=' + target.getAttribute('data-id') +
         '&playlist_id=' + option.getAttribute('data-plid');
 
@@ -21,7 +21,7 @@ function add_playlist_item(target) {
     var tile = target.parentNode.parentNode.parentNode.parentNode.parentNode;
     tile.style.display = 'none';
 
-    var url = '/playlist_ajax?action_add_video=1&redirect=false' +
+    var url = '/playlist_ajax?action=add_video&redirect=false' +
         '&video_id=' + target.getAttribute('data-id') +
         '&playlist_id=' + target.getAttribute('data-plid');
 
@@ -36,7 +36,7 @@ function remove_playlist_item(target) {
     var tile = target.parentNode.parentNode.parentNode.parentNode.parentNode;
     tile.style.display = 'none';
 
-    var url = '/playlist_ajax?action_remove_video=1&redirect=false' +
+    var url = '/playlist_ajax?action=remove_video&redirect=false' +
         '&set_video_id=' + target.getAttribute('data-index') +
         '&playlist_id=' + target.getAttribute('data-plid');
 

assets/js/subscribe_widget.js

@@ -16,7 +16,7 @@ function subscribe() {
     subscribe_button.onclick = unsubscribe;
     subscribe_button.innerHTML = '<b>' + subscribe_data.unsubscribe_text + ' | ' + subscribe_data.sub_count_text + '</b>';
 
-    var url = '/subscription_ajax?action_create_subscription_to_channel=1&redirect=false' +
+    var url = '/subscription_ajax?action=create_subscription_to_channel&redirect=false' +
         '&c=' + subscribe_data.ucid;
 
     helpers.xhr('POST', url, {payload: payload, retries: 5, entity_name: 'subscribe request'}, {
@@ -32,7 +32,7 @@ function unsubscribe() {
     subscribe_button.onclick = subscribe;
     subscribe_button.innerHTML = '<b>' + subscribe_data.subscribe_text + ' | ' + subscribe_data.sub_count_text + '</b>';
 
-    var url = '/subscription_ajax?action_remove_subscriptions=1&redirect=false' +
+    var url = '/subscription_ajax?action=remove_subscriptions&redirect=false' +
         '&c=' + subscribe_data.ucid;
 
     helpers.xhr('POST', url, {payload: payload, retries: 5, entity_name: 'unsubscribe request'}, {

assets/js/watch.js

@@ -67,6 +67,10 @@ function get_playlist(plid) {
             '&format=html&hl=' + video_data.preferences.locale;
     }
 
+    if (video_data.params.listen) {
+        plid_url += '&listen=1'
+    }
+
     helpers.xhr('GET', plid_url, {retries: 5, entity_name: 'playlist'}, {
         on200: function (response) {
             playlist.innerHTML = response.playlistHtml;

assets/js/watched_widget.js

@@ -6,7 +6,7 @@ function mark_watched(target) {
     var tile = target.parentNode.parentNode.parentNode.parentNode.parentNode;
     tile.style.display = 'none';
 
-    var url = '/watch_ajax?action_mark_watched=1&redirect=false' +
+    var url = '/watch_ajax?action=mark_watched&redirect=false' +
         '&id=' + target.getAttribute('data-id');
 
     helpers.xhr('POST', url, {payload: payload}, {
@@ -22,7 +22,7 @@ function mark_unwatched(target) {
     var count = document.getElementById('count');
     count.textContent--;
 
-    var url = '/watch_ajax?action_mark_unwatched=1&redirect=false' +
+    var url = '/watch_ajax?action=mark_unwatched&redirect=false' +
         '&id=' + target.getAttribute('data-id');
 
     helpers.xhr('POST', url, {payload: payload}, {

config/config.example.yml

@@ -54,6 +54,53 @@ db:
 ##
 #signature_server:
 
+##
+## Invidious companion is an external program
+## for loading the video streams from YouTube servers.
+##
+## When this setting is commented out, Invidious companion is not used.
+## Otherwise, Invidious will proxy the requests to Invidious companion.
+## 
+## Note: multiple URL can be configured. In this case, invidious will
+## randomly pick one every time video data needs to be retrieved. This
+## URL is then kept in the video metadata cache to allow video playback
+## to work. Once said cache has expired, requesting that video's data
+## again will cause a new companion URL to be picked.
+##
+## The parameter private_url needs to be configured for the internal
+## communication between the companion and Invidious.
+## And public_url is the public URL from which companion is listening
+## to the requests from the user(s).
+##
+## If you are using a reverse proxy then you will probably need to
+## configure the public_url to be the same as the domain used for Invidious.
+## Also apply when used from an external IP address (without a domain).
+## Examples: https://MYINVIDIOUSDOMAIN or http://192.168.1.100:8282
+##
+## Both parameter can have identical URL when Invidious is hosted in
+## an internal network or at home or locally (localhost).
+##
+## Accepted values: "http(s)://<IP-HOSTNAME>:<Port>"
+## Default: <none>
+##
+#invidious_companion:
+#  - private_url: "http://localhost:8282"
+#    public_url: "http://localhost:8282"
+
+##
+## API key for Invidious companion, used for securing the communication
+## between Invidious and Invidious companion.
+## The size of the key needs to be more or equal to 16.
+##
+## Note: This parameter is mandatory when Invidious companion is enabled
+## and should be a random string.
+## Such random string can be generated on linux with the following
+## command: `pwgen 16 1`
+##
+## Accepted values: a string
+## Default: <none>
+##
+#invidious_companion_key: "CHANGE_ME!!"
 
 #########################################
 #
@@ -178,11 +225,11 @@ https_only: false
 ##
 ## If unset, then no HTTP proxy will be used.
 ## 
-http_proxy:
+#http_proxy:
-  user:
+#  user:
-  password:
+#  password:
-  host:
+#  host:
-  port:
+#  port:
 
 
 ##
@@ -237,9 +284,11 @@ http_proxy:
 ## Enables colors in logs. Useful for debugging purposes
 ## This is overridden if "-k" or "--colorize"
 ## are passed on the command line.
+## Colors are also disabled if the environment variable
+## NO_COLOR is present and has any value
 ##
 ## Accepted values: true, false
-## Default: false
+## Default: true
 ##
 #colorize_logs: false
 

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM crystallang/crystal:1.14.0-alpine AS builder
+FROM mirror.gcr.io/crystallang/crystal:1.14.0-alpine AS builder
 
 RUN apk add --no-cache sqlite-static yaml-static
 
@@ -23,7 +23,7 @@ RUN crystal spec --warnings all \
     --link-flags "-lxml2 -llzma"
 RUN if [[ "${release}" == 1 ]] ; then \
         crystal build ./src/invidious.cr \
-        --release --mcpu=x86-64-v3 \
+        --release --mcpu=x86-64-v2 \
         --static --warnings all \
         --link-flags "-lxml2 -llzma"; \
     else \
@@ -32,8 +32,8 @@ RUN if [[ "${release}" == 1 ]] ; then \
         --link-flags "-lxml2 -llzma"; \
     fi
 
-FROM alpine:3.18
+FROM mirror.gcr.io/alpine:3.20
-RUN apk add --no-cache rsvg-convert ttf-opensans tini
+RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata
 WORKDIR /invidious
 RUN addgroup -g 1000 -S invidious && \
     adduser -u 1000 -S invidious -G invidious

docker/Dockerfile.arm64

@@ -1,5 +1,6 @@
-FROM alpine:3.19 AS builder
+FROM alpine:3.20 AS builder
-RUN apk add --no-cache 'crystal=1.10.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-static zlib-static openssl-libs-static openssl-dev musl-dev xz-static
+RUN apk add --no-cache 'crystal=1.12.2-r0' shards sqlite-static yaml-static yaml-dev libxml2-static \
+       zlib-static openssl-libs-static openssl-dev musl-dev xz-static
 
 ARG release
 
@@ -32,8 +33,8 @@ RUN if [[ "${release}" == 1 ]] ; then \
         --link-flags "-lxml2 -llzma"; \
     fi
 
-FROM alpine:3.18
+FROM alpine:3.20
-RUN apk add --no-cache rsvg-convert ttf-opensans tini
+RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata
 WORKDIR /invidious
 RUN addgroup -g 1000 -S invidious && \
     adduser -u 1000 -S invidious -G invidious

shard.lock

@@ -14,7 +14,7 @@ shards:
 
   db:
     git: https://github.com/crystal-lang/crystal-db.git
-    version: 0.10.1
+    version: 0.13.1
 
   exception_page:
     git: https://github.com/crystal-loot/exception_page.git
@@ -37,7 +37,7 @@ shards:
 
   pg:
     git: https://github.com/will/crystal-pg.git
-    version: 0.24.0
+    version: 0.28.0
 
   pool:
     git: https://github.com/ysbaddaden/pool.git
@@ -61,5 +61,5 @@ shards:
 
   sqlite3:
     git: https://github.com/crystal-lang/crystal-sqlite3.git
-    version: 0.18.0
+    version: 0.21.0
 

shard.yml

@@ -1,21 +1,20 @@
 name: invidious
-version: 0.20.1
+version: 2.20241110.0-dev
 
 authors:
-  - Omar Roth <omarroth@protonmail.com>
+  - Invidious team <contact@invidious.io>
-  - Invidious team
+  - Contributors!
 
-targets:
+description: |
-  invidious:
+   Invidious is an alternative front-end to YouTube 
-    main: src/invidious.cr
 
 dependencies:
   pg:
     github: will/crystal-pg
-    version: ~> 0.24.0
+    version: ~> 0.28.0
   sqlite3:
     github: crystal-lang/crystal-sqlite3
-    version: ~> 0.18.0
+    version: ~> 0.21.0
   kemal:
     github: kemalcr/kemal
     version: ~> 1.1.2
@@ -45,6 +44,10 @@ development_dependencies:
     github: crystal-ameba/ameba
     version: ~> 1.6.1
 
-crystal: ">= 1.0.0, < 2.0.0"
+crystal: ">= 1.10.0, < 2.0.0"
 
 license: AGPLv3
+
+repository: https://github.com/iv-org/invidious
+homepage: https://invidious.io
+documentation: https://docs.invidious.io

src/invidious.cr

@@ -88,7 +88,6 @@ REDDIT_URL      = URI.parse("https://www.reddit.com")
 YT_URL          = URI.parse("https://www.youtube.com")
 PUBSUB_HOST_URL = CONFIG.pubsub_domain
 HOST_URL        = make_host_url(Kemal.config)
-EXT_VIDEOP_LIST = gen_videoplayback_proxy_list()
 
 CHARS_SAFE         = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
 TEST_IDS           = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
@@ -211,10 +210,16 @@ Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new
 
 if !CONFIG.external_videoplayback_proxy.empty?
   Invidious::Jobs.register Invidious::Jobs::CheckExternalProxy.new
+else
+  # Invidious will it's own videoplayback proxy unless the admin decides to rewrite
+  # the /videoplayback location in the reverse proxy configuration (NGINX, Caddy, etc)
+  LOGGER.info("jobs: Disabling CheckExternalProxy job. Invidious will it's own videoplayback proxy")
 end
 
-if CONFIG.refresh_tokens
+if !CONFIG.tokens_server.empty?
-  Invidious::Jobs.register Invidious::Jobs::RefreshTokens.new
+  Invidious::Jobs.register Invidious::Jobs::RefreshSessionTokens.new
+else
+  LOGGER.info("jobs: Disabling RefreshSessionTokens job. Invidious will use the tokens that are on the configuration file")
 end
 
 Invidious::Jobs.start_all

src/invidious/channels/videos.cr

@@ -1,95 +1,3 @@
-def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
-  object_inner_2 = {
-    "2:0:embedded" => {
-      "1:0:varint" => 0_i64,
-    },
-    "5:varint"  => 50_i64,
-    "6:varint"  => 1_i64,
-    "7:varint"  => (page * 30).to_i64,
-    "9:varint"  => 1_i64,
-    "10:varint" => 0_i64,
-  }
-
-  object_inner_2_encoded = object_inner_2
-    .try { |i| Protodec::Any.cast_json(i) }
-    .try { |i| Protodec::Any.from_json(i) }
-    .try { |i| Base64.urlsafe_encode(i) }
-    .try { |i| URI.encode_www_form(i) }
-
-  content_type_numerical =
-    case content_type
-    when "videos"      then 15
-    when "livestreams" then 14
-    else                    15 # Fallback to "videos"
-    end
-
-  sort_type_numerical =
-    case content_type
-    when "videos"      then 3
-    when "livestreams" then 5
-    else                    3 # Fallback to "videos"
-    end
-
-  if content_type == "livestreams"
-    sort_by_numerical =
-      case sort_by
-      when "newest"  then 12_i64
-      when "popular" then 14_i64
-      when "oldest"  then 13_i64
-      else                12_i64 # Fallback to "newest"
-      end
-  else
-    sort_by_numerical =
-      case sort_by
-      when "newest"  then 1_i64
-      when "popular" then 2_i64
-      when "oldest"  then 4_i64
-      else                1_i64 # Fallback to "newest"
-      end
-  end
-
-  object_inner_1 = {
-    "110:embedded" => {
-      "3:embedded" => {
-        "#{content_type_numerical}:embedded" => {
-          "1:embedded" => {
-            "1:string" => object_inner_2_encoded,
-          },
-          "2:embedded" => {
-            "1:string" => "00000000-0000-0000-0000-000000000000",
-          },
-          "#{sort_type_numerical}:varint" => sort_by_numerical,
-        },
-      },
-    },
-  }
-
-  object_inner_1_encoded = object_inner_1
-    .try { |i| Protodec::Any.cast_json(i) }
-    .try { |i| Protodec::Any.from_json(i) }
-    .try { |i| Base64.urlsafe_encode(i) }
-    .try { |i| URI.encode_www_form(i) }
-
-  object = {
-    "80226972:embedded" => {
-      "2:string"  => ucid,
-      "3:string"  => object_inner_1_encoded,
-      "35:string" => "browse-feed#{ucid}videos102",
-    },
-  }
-
-  continuation = object.try { |i| Protodec::Any.cast_json(i) }
-    .try { |i| Protodec::Any.from_json(i) }
-    .try { |i| Base64.urlsafe_encode(i) }
-    .try { |i| URI.encode_www_form(i) }
-
-  return continuation
-end
-
-def make_initial_content_ctoken(ucid, content_type, sort_by) : String
-  return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by)
-end
-
 module Invidious::Channel::Tabs
   extend self
 
@@ -118,7 +26,7 @@ module Invidious::Channel::Tabs
   end
 
   def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
-    continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by)
+    continuation ||= make_initial_videos_ctoken(ucid, sort_by)
     initial_data = YoutubeAPI.browse(continuation: continuation)
 
     return extract_items(initial_data, author, ucid)
@@ -147,14 +55,10 @@ module Invidious::Channel::Tabs
   #  Shorts
   # -------------------
 
-  def get_shorts(channel : AboutChannel, continuation : String? = nil)
+  def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
-    if continuation.nil?
+    continuation ||= make_initial_shorts_ctoken(channel.ucid, sort_by)
-      # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
-      # TODO: try to extract the continuation tokens that allows other sorting options
-      initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
-    else
     initial_data = YoutubeAPI.browse(continuation: continuation)
-    end
+
     return extract_items(initial_data, channel.author, channel.ucid)
   end
 
@@ -162,9 +66,8 @@ module Invidious::Channel::Tabs
   #  Livestreams
   # -------------------
 
-  def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest")
+  def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
-    continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by)
+    continuation ||= make_initial_livestreams_ctoken(channel.ucid, sort_by)
-
     initial_data = YoutubeAPI.browse(continuation: continuation)
 
     return extract_items(initial_data, channel.author, channel.ucid)
@@ -188,4 +91,102 @@ module Invidious::Channel::Tabs
 
     return items, next_continuation
   end
+
+  # -------------------
+  #  C-tokens
+  # -------------------
+
+  private def sort_options_videos_short(sort_by : String)
+    case sort_by
+    when "newest"  then return 4_i64
+    when "popular" then return 2_i64
+    when "oldest"  then return 5_i64
+    else                return 4_i64 # Fallback to "newest"
+    end
+  end
+
+  # Generate the initial "continuation token" to get the first page of the
+  # "videos" tab. The following page requires the ctoken provided in that
+  # first page, and so on.
+  private def make_initial_videos_ctoken(ucid : String, sort_by = "newest")
+    object = {
+      "15:embedded" => {
+        "2:embedded" => {
+          "1:string" => "00000000-0000-0000-0000-000000000000",
+        },
+        "4:varint" => sort_options_videos_short(sort_by),
+      },
+    }
+
+    return channel_ctoken_wrap(ucid, object)
+  end
+
+  # Generate the initial "continuation token" to get the first page of the
+  # "shorts" tab. The following page requires the ctoken provided in that
+  # first page, and so on.
+  private def make_initial_shorts_ctoken(ucid : String, sort_by = "newest")
+    object = {
+      "10:embedded" => {
+        "2:embedded" => {
+          "1:string" => "00000000-0000-0000-0000-000000000000",
+        },
+        "4:varint" => sort_options_videos_short(sort_by),
+      },
+    }
+
+    return channel_ctoken_wrap(ucid, object)
+  end
+
+  # Generate the initial "continuation token" to get the first page of the
+  # "livestreams" tab. The following page requires the ctoken provided in that
+  # first page, and so on.
+  private def make_initial_livestreams_ctoken(ucid : String, sort_by = "newest")
+    sort_by_numerical =
+      case sort_by
+      when "newest"  then 12_i64
+      when "popular" then 14_i64
+      when "oldest"  then 13_i64
+      else                12_i64 # Fallback to "newest"
+      end
+
+    object = {
+      "14:embedded" => {
+        "2:embedded" => {
+          "1:string" => "00000000-0000-0000-0000-000000000000",
+        },
+        "5:varint" => sort_by_numerical,
+      },
+    }
+
+    return channel_ctoken_wrap(ucid, object)
+  end
+
+  # The protobuf structure common between videos/shorts/livestreams
+  private def channel_ctoken_wrap(ucid : String, object)
+    object_inner = {
+      "110:embedded" => {
+        "3:embedded" => object,
+      },
+    }
+
+    object_inner_encoded = object_inner
+      .try { |i| Protodec::Any.cast_json(i) }
+      .try { |i| Protodec::Any.from_json(i) }
+      .try { |i| Base64.urlsafe_encode(i) }
+      .try { |i| URI.encode_www_form(i) }
+
+    object = {
+      "80226972:embedded" => {
+        "2:string" => ucid,
+        "3:string" => object_inner_encoded,
+      },
+    }
+
+    continuation = object.try { |i| Protodec::Any.cast_json(i) }
+      .try { |i| Protodec::Any.from_json(i) }
+      .try { |i| Base64.urlsafe_encode(i) }
+      .try { |i| URI.encode_www_form(i) }
+
+    return continuation
+  end
 end

src/invidious/config.cr

@@ -45,8 +45,6 @@ struct ConfigPreferences
   property vr_mode : Bool = true
   property show_nick : Bool = true
   property save_player_pos : Bool = false
-  property po_token : String = ""
-  property visitor_data : String = ""
 
   def to_tuple
     {% begin %}
@@ -69,6 +67,16 @@ end
 class Config
   include YAML::Serializable
 
+  class CompanionConfig
+    include YAML::Serializable
+
+    @[YAML::Field(converter: Preferences::URIConverter)]
+    property private_url : URI = URI.parse("")
+
+    @[YAML::Field(converter: Preferences::URIConverter)]
+    property public_url : URI = URI.parse("")
+  end
+
   # Number of threads to use for crawling videos from channels (for updating subscriptions)
   property channel_threads : Int32 = 1
   # Time interval between two executions of the job that crawls channel videos (subscriptions update).
@@ -108,12 +116,12 @@ class Config
   property materialious_domain : String?
   # Alternative domains. You can add other domains, like TOR and I2P addresses
   property alternative_domains : Array(String) = [] of String
-  property donation_url : String?
+  # Backend domains. Domains for numbered backends
-  property contact_url : String?
+  property backend_domains : Array(String) = [] of String
-  property home_domain : String?
 
   # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
   property use_pubsub_feeds : Bool | Int32 = false
+  property use_innertube_for_feeds : Bool = true
   property popular_enabled : Bool = true
   property captcha_enabled : Bool = true
   property login_enabled : Bool = true
@@ -177,6 +185,12 @@ class Config
   # poToken for passing bot attestation
   property po_token : String? = nil
 
+  # Invidious companion
+  property invidious_companion : Array(CompanionConfig) = [] of CompanionConfig
+
+  # Invidious companion API key
+  property invidious_companion_key : String = ""
+
   # Saved cookies in "name1=value1; name2=value2..." format
   @[YAML::Field(converter: Preferences::StringToCookies)]
   property cookies : HTTP::Cookies = HTTP::Cookies.new
@@ -195,15 +209,16 @@ class Config
 
   # External videoplayback proxies list. They should include `https://`
   # at the start of the URI
-  property external_videoplayback_proxy : Array(NamedTuple(url: String, balance: Bool)) = [] of NamedTuple(url: String, balance: Bool)
+  property external_videoplayback_proxy : Array(String) = [] of String
-
-  # Job to refresh tokens from a Redis compatible DB
-  property refresh_tokens : Bool = true
 
   property pubsub_domain : String = ""
 
   property ignore_user_tokens : Bool = false
 
+  property server_id_cookie_name : String = "INVIDIOUS_SERVER_ID"
+
+  property tokens_server : String = ""
+
   {% if flag?(:linux) %}
     property reload_config_automatically : Bool = true
   {% end %}
@@ -292,6 +307,9 @@ class Config
     config = Config.from_yaml(config_yaml)
 
     # Update config from env vars (upcased and prefixed with "INVIDIOUS_")
+    #
+    # Also checks if any top-level config options are set to "CHANGE_ME!!"
+    # TODO: Support non-top-level config options such as the ones in DBConfig
     {% for ivar in Config.instance_vars %}
         {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %}
 
@@ -328,16 +346,36 @@ class Config
                 exit(1)
             end
         end
+
+        # Warn when any config attribute is set to "CHANGE_ME!!"
+        if config.{{ivar.id}} == "CHANGE_ME!!"
+          puts "Config: The value of '#{ {{ivar.stringify}} }' needs to be changed!!"
+          exit(1)
+        end
     {% end %}
 
+    if config.invidious_companion.present?
+      # invidious_companion and signature_server can't work together
+      if config.signature_server
+        puts "Config: You can not run inv_sig_helper and invidious_companion at the same time."
+        exit(1)
+      elsif config.invidious_companion_key.empty?
+        puts "Config: Please configure a key if you are using invidious companion."
+        exit(1)
+      elsif config.invidious_companion_key == "CHANGE_ME!!"
+        puts "Config: The value of 'invidious_companion_key' needs to be changed!!"
+        exit(1)
+      elsif config.invidious_companion_key.size < 16
+        puts "Config: The value of 'invidious_companion_key' needs to be a size of 16 or more."
+        exit(1)
+      end
+    end
+
     # HMAC_key is mandatory
     # See: https://github.com/iv-org/invidious/issues/3854
     if config.hmac_key.empty?
       puts "Config: 'hmac_key' is required/can't be empty"
       exit(1)
-    elsif config.hmac_key == "CHANGE_ME!!"
-      puts "Config: The value of 'hmac_key' needs to be changed!!"
-      exit(1)
     end
 
     # Build database_url from db.* if it's not set directly

src/invidious/database/videos.cr

@@ -4,22 +4,11 @@ module Invidious::Database::Videos
   extend self
 
   def insert(video : Video)
-    request = <<-SQL
-      INSERT INTO videos
-      VALUES ($1, $2, $3)
-      ON CONFLICT (id) DO NOTHING
-    SQL
-
     REDIS_DB.set(video.id, video.info.to_json, ex: 14400)
     REDIS_DB.set(video.id + ":time", video.updated, ex: 14400)
   end
 
   def delete(id)
-    request = <<-SQL
-      DELETE FROM videos *
-      WHERE id = $1
-    SQL
-
     REDIS_DB.del(id)
     REDIS_DB.del(id + ":time")
   end
@@ -44,11 +33,6 @@ module Invidious::Database::Videos
   end
 
   def select(id : String) : Video?
-    request = <<-SQL
-      SELECT * FROM videos
-      WHERE id = $1
-    SQL
-
     if ((info = REDIS_DB.get(id)) && (time = REDIS_DB.get(id + ":time")))
       return Video.new({
         id:      id,

src/invidious/frontend/watch_page.cr

@@ -13,7 +13,7 @@ module Invidious::Frontend::WatchPage
       @full_videos,
       @video_streams,
       @audio_streams,
-      @captions
+      @captions,
     )
     end
   end

src/invidious/helpers/crystal_class_overrides.cr

@@ -18,40 +18,6 @@ end
 class HTTP::Client
   property family : Socket::Family = Socket::Family::UNSPEC
 
-  # Override stdlib to automatically initialize proxy if configured
-  #
-  # Accurate as of crystal 1.12.1
-
-  def initialize(@host : String, port = nil, tls : TLSContext = nil)
-    check_host_only(@host)
-
-    {% if flag?(:without_openssl) %}
-      if tls
-        raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time"
-      end
-      @tls = nil
-    {% else %}
-      @tls = case tls
-             when true
-               OpenSSL::SSL::Context::Client.new
-             when OpenSSL::SSL::Context::Client
-               tls
-             when false, nil
-               nil
-             end
-    {% end %}
-
-    @port = (port || (@tls ? 443 : 80)).to_i
-
-    self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
-  end
-
-  def initialize(@io : IO, @host = "", @port = 80)
-    @reconnect = false
-
-    self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
-  end
-
   private def io
     io = @io
     return io if io

src/invidious/helpers/errors.cr

@@ -130,7 +130,7 @@ def error_json_helper(
   env : HTTP::Server::Context,
   status_code : Int32,
   exception : Exception,
-  additional_fields : Hash(String, Object) | Nil = nil
+  additional_fields : Hash(String, Object) | Nil = nil,
 )
   if exception.is_a?(InfoException)
     return error_json_helper(env, status_code, exception.message || "", additional_fields)
@@ -152,7 +152,7 @@ def error_json_helper(
   env : HTTP::Server::Context,
   status_code : Int32,
   message : String,
-  additional_fields : Hash(String, Object) | Nil = nil
+  additional_fields : Hash(String, Object) | Nil = nil,
 )
   env.response.content_type = "application/json"
   env.response.status_code = status_code
@@ -180,6 +180,7 @@ def error_redirect_helper(env : HTTP::Server::Context)
     next_steps_text = translate(locale, "next_steps_error_message")
     refresh = translate(locale, "next_steps_error_message_refresh")
     go_to_youtube = translate(locale, "next_steps_error_message_go_to_youtube")
+    go_to_youtube_embed = translate(locale, "videoinfo_youTube_embed_link")
     switch_instance = translate(locale, "Switch Invidious Instance")
 
     return <<-END_HTML
@@ -193,6 +194,7 @@ def error_redirect_helper(env : HTTP::Server::Context)
         </li>
         <li>
           <a rel="noreferrer noopener" href="https://youtube.com#{env.request.resource}">#{go_to_youtube}</a>
+          (<a rel="noreferrer noopener" href="https://youtube.com/embed/#{env.params.query["v"]}">#{go_to_youtube_embed}</a>)
         </li>
       </ul>
     END_HTML

src/invidious/helpers/handlers.cr

@@ -27,6 +27,7 @@ class Kemal::RouteHandler
   # Processes the route if it's a match. Otherwise renders 404.
   private def process_request(context)
     raise Kemal::Exceptions::RouteNotFound.new(context) unless context.route_found?
+    return if context.response.closed?
     content = context.route.handler.call(context)
 
     if !Kemal.config.error_handlers.empty? && Kemal.config.error_handlers.has_key?(context.response.status_code) && exclude_match?(context)

src/invidious/helpers/i18n.cr

@@ -1,8 +1,22 @@
+# Languages requiring a better level of translation (at least 20%)
+# to be added to the list below:
+#
+#  "af"      => "", # Afrikaans
+#  "az"      => "", # Azerbaijani
+#  "be"      => "", # Belarusian
+#  "bn_BD"   => "", # Bengali (Bangladesh)
+#  "ia"      => "", # Interlingua
+#  "or"      => "", # Odia
+#  "tk"      => "", # Turkmen
+#  "tok      => "", # Toki Pona
+#
 LOCALES_LIST = {
   "ar"      => "العربية",               # Arabic
+  "bg"      => "български",             # Bulgarian
   "bn"      => "বাংলা",                 # Bengali
   "ca"      => "Català",                # Catalan
   "cs"      => "Čeština",               # Czech
+  "cy"      => "Cymraeg",               # Welsh
   "da"      => "Dansk",                 # Danish
   "de"      => "Deutsch",               # German
   "el"      => "Ελληνικά",              # Greek
@@ -23,6 +37,7 @@ LOCALES_LIST = {
   "it"      => "Italiano",              # Italian
   "ja"      => "日本語",                   # Japanese
   "ko"      => "한국어",                   # Korean
+  "lmo"     => "Lombard",               # Lombard
   "lt"      => "Lietuvių",              # Lithuanian
   "nb-NO"   => "Norsk bokmål",          # Norwegian Bokmål
   "nl"      => "Nederlands",            # Dutch

src/invidious/helpers/logger.cr

@@ -12,7 +12,9 @@ enum LogLevel
 end
 
 class Invidious::LogHandler < Kemal::BaseLogHandler
-  def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, @color : Bool = true)
+  def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true)
+    Colorize.enabled = use_color
+    Colorize.on_tty_only!
   end
 
   def call(context : HTTP::Server::Context)
@@ -56,8 +58,7 @@ class Invidious::LogHandler < Kemal::BaseLogHandler
   {% for level in %w(trace debug info warn error fatal) %}
     def {{level.id}}(message : String)
       if LogLevel::{{level.id.capitalize}} >= @level
-        puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})).toggle(@color))
+        puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})))
-
       end
     end
   {% end %}

src/invidious/helpers/redis_tokens.cr

@@ -1,54 +0,0 @@
-module Tokens
-  extend self
-  @@po_token : String | Nil
-  @@visitor_data : String | Nil
-
-  def refresh_tokens
-    @@po_token = REDIS_DB.get("invidious:po_token")
-    @@visitor_data = REDIS_DB.get("invidious:visitor_data")
-	if !@@po_token.nil? && !@@visitor_data.nil?
-		LOGGER.debug("RefreshTokens: Successfully updated tokens")
-	else
-		LOGGER.warn("RefreshTokens: Tokens are empty!")
-	end
-    LOGGER.trace("RefreshTokens: Tokens are:")
-    LOGGER.trace("RefreshTokens: po_token: #{@@po_token}")
-    LOGGER.trace("RefreshTokens: visitor_data: #{@@visitor_data}")
-  end
-
-  def get_tokens
-    return {@@po_token, @@visitor_data}
-  end
-
-  def get_po_token
-    return @@po_token
-  end
-
-  def get_visitor_data
-    return @@visitor_data
-  end
-
-  def generate_tokens(user : String)
-    po_token = ""
-    visitor_data = ""
-    attempts = 0
-
-    LOGGER.debug("Generating po_token and visitor_data for user: '#{user}'")
-    REDIS_DB.publish("generate-token", "#{user}")
-
-    while REDIS_DB.get("invidious:#{user}:po_token").nil? && REDIS_DB.get("invidious:#{user}:visitor_data").nil?
-      if attempts > 50
-        break
-      end
-      LOGGER.debug("Waiting for tokens to arrive at redis for user: '#{user}'")
-      attempts += 1
-      sleep 250.milliseconds
-    end
-
-    po_token = REDIS_DB.get("invidious:#{user}:po_token")
-    visitor_data = REDIS_DB.get("invidious:#{user}:visitor_data")
-
-    LOGGER.debug("Tokens successfully generated for user: '#{user}'")
-    return {po_token, visitor_data}
-  end
-end

src/invidious/helpers/serialized_yt_data.cr

@@ -24,6 +24,7 @@ struct SearchVideo
   property length_seconds : Int32
   property premiere_timestamp : Time?
   property author_verified : Bool
+  property author_thumbnail : String?
   property badges : VideoBadges
 
   def to_xml(auto_generated, query_params, xml : XML::Builder)
@@ -88,6 +89,24 @@ struct SearchVideo
       json.field "authorUrl", "/channel/#{self.ucid}"
       json.field "authorVerified", self.author_verified
 
+      author_thumbnail = self.author_thumbnail
+
+      if author_thumbnail
+        json.field "authorThumbnails" do
+          json.array do
+            qualities = {32, 48, 76, 100, 176, 512}
+
+            qualities.each do |quality|
+              json.object do
+                json.field "url", author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
+                json.field "width", quality
+                json.field "height", quality
+              end
+            end
+          end
+        end
+      end
+
       json.field "videoThumbnails" do
         Invidious::JSONify::APIv1.thumbnails(json, self.id)
       end
@@ -223,7 +242,7 @@ struct SearchChannel
 
           qualities.each do |quality|
             json.object do
-              json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
+              json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
               json.field "width", quality
               json.field "height", quality
             end

src/invidious/helpers/session_tokens.cr

@@ -0,0 +1,35 @@
+module SessionTokens
+  extend self
+  @@po_token : String | Nil
+  @@visitor_data : String | Nil
+
+  def refresh_tokens
+    begin
+      response = HTTP::Client.get "#{CONFIG.tokens_server}/generate"
+      if !response.status_code == 200
+        LOGGER.error("RefreshSessionTokens: Expected response to have status code 200 but got #{response.status_code} from #{CONFIG.tokens_server}")
+      end
+      json = JSON.parse(response.body)
+      @@po_token = json.try &.["potoken"].as_s || nil
+      @@visitor_data = json.try &.["visitorData"].as_s || nil
+    rescue ex
+      LOGGER.error("RefreshSessionTokens: Failed to fetch tokens from #{CONFIG.tokens_server}: #{ex.message}")
+      return
+    end
+
+    if !@@po_token.nil? && !@@visitor_data.nil?
+      set_tokens
+      LOGGER.debug("RefreshSessionTokens: Successfully updated po_token and visitor_data")
+    else
+      LOGGER.warn("RefreshSessionTokens: Tokens are empty!. Invidious will use the tokens that are on the configuration file")
+    end
+    LOGGER.trace("RefreshSessionTokens: Tokens are:")
+    LOGGER.trace("RefreshSessionTokens: po_token: #{CONFIG.po_token}")
+    LOGGER.trace("RefreshSessionTokens: visitor_data: #{CONFIG.visitor_data}")
+  end
+
+  def set_tokens
+    CONFIG.po_token = @@po_token
+    CONFIG.visitor_data = @@visitor_data
+  end
+end

src/invidious/helpers/utils.cr

@@ -384,16 +384,21 @@ def parse_link_endpoint(endpoint : JSON::Any, text : String, video_id : String)
   return text
 end
 
-# Generates a list of external videoplayback proxies for 
+def encrypt_ecb_without_salt(data, key)
-# CSP
+  cipher = OpenSSL::Cipher.new("aes-128-ecb")
-def gen_videoplayback_proxy_list
+  cipher.encrypt
-  if !CONFIG.external_videoplayback_proxy.empty?
+  cipher.key = key
-    external_videoplayback_proxy = ""
+
-    CONFIG.external_videoplayback_proxy.each do |proxy|
+  io = IO::Memory.new
-      external_videoplayback_proxy += " #{proxy[:url]}"
+  io.write(cipher.update(data))
+  io.write(cipher.final)
+  io.rewind
+
+  return io
 end
-  else
+
-    external_videoplayback_proxy = ""
+def invidious_companion_encrypt(data)
-  end
+  timestamp = Time.utc.to_unix
-  return external_videoplayback_proxy
+  encrypted_data = encrypt_ecb_without_salt("#{timestamp}|#{data}", CONFIG.invidious_companion_key)
+  return Base64.urlsafe_encode(encrypted_data)
 end

src/invidious/http_server/utils.cr

@@ -9,14 +9,14 @@ module Invidious::HttpServer
     def check_external_proxy
       CONFIG.external_videoplayback_proxy.each do |proxy|
         begin
-          response = HTTP::Client.get("#{proxy[:url]}/health")
+          response = HTTP::Client.get("#{proxy}/health")
           if response.status_code == 200
-            @@proxy_alive = proxy[:url]
+            @@proxy_alive = proxy
-            LOGGER.debug("CheckExternalProxy: Proxy set to: '#{proxy[:url]}'")
+            LOGGER.debug("CheckExternalProxy: Proxy set to: '#{proxy}'")
             break
           end
         rescue
-          LOGGER.debug("CheckExternalProxy: Proxy '#{proxy[:url]}' is not available")
+          LOGGER.debug("CheckExternalProxy: Proxy '#{proxy}' is not available")
         end
       end
       if @@proxy_alive.empty?

src/invidious/jobs/refresh_tokens.cr

@@ -1,10 +1,10 @@
-class Invidious::Jobs::RefreshTokens < Invidious::Jobs::BaseJob
+class Invidious::Jobs::RefreshSessionTokens < Invidious::Jobs::BaseJob
   def initialize
   end
 
   def begin
     loop do
-      Tokens.refresh_tokens
+      SessionTokens.refresh_tokens
       LOGGER.info("RefreshTokens: Done, sleeping for 5 seconds")
       sleep 5.seconds
       Fiber.yield

src/invidious/jsonify/api_v1/common.cr

@@ -8,7 +8,7 @@ module Invidious::JSONify::APIv1
       build_thumbnails(id).each do |thumbnail|
         json.object do
           json.field "quality", thumbnail[:name]
-          json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg"
+          json.field "url", "/vi/#{id}/#{thumbnail["url"]}.jpg"
           json.field "width", thumbnail[:width]
           json.field "height", thumbnail[:height]
         end

src/invidious/jsonify/api_v1/video_json.cr

@@ -267,6 +267,12 @@ module Invidious::JSONify::APIv1
                 json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
                 json.field "viewCountText", rv["short_view_count"]?
                 json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
+                json.field "published", rv["published"]?
+                if !rv["published"]?.nil?
+                  json.field "publishedText", translate(locale, "`x` ago", recode_date(Time.parse_rfc3339(rv["published"].to_s), locale))
+                else
+                  json.field "publishedText", ""
+                end
               end
             end
           end

src/invidious/mixes.cr

@@ -81,7 +81,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
   })
 end
 
-def template_mix(mix)
+def template_mix(mix, listen)
   html = <<-END_HTML
   <h3>
     <a href="/mix?list=#{mix["mixId"]}">
@@ -95,7 +95,7 @@ def template_mix(mix)
   mix["videos"].as_a.each do |video|
     html += <<-END_HTML
       <li class="pure-menu-item">
-        <a href="/watch?v=#{video["videoId"]}&list=#{mix["mixId"]}">
+        <a href="/watch?v=#{video["videoId"]}&list=#{mix["mixId"]}#{listen ? "&listen=1" : ""}">
           <div class="thumbnail">
               <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" />
               <p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p>

src/invidious/playlists.cr

@@ -505,7 +505,7 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
   return videos
 end
 
-def template_playlist(playlist)
+def template_playlist(playlist, listen)
   html = <<-END_HTML
   <h3>
     <a href="/playlist?list=#{playlist["playlistId"]}">
@@ -519,7 +519,7 @@ def template_playlist(playlist)
   playlist["videos"].as_a.each do |video|
     html += <<-END_HTML
       <li class="pure-menu-item" id="#{video["videoId"]}">
-        <a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}">
+        <a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}#{listen ? "&listen=1" : ""}">
           <div class="thumbnail">
               <img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" />
               <p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p>

src/invidious/routes/account.cr

@@ -326,17 +326,9 @@ module Invidious::Routes::Account
       end
     end
 
-    if env.params.query["action_revoke_token"]?
+    case action = env.params.query["action"]?
-      action = "action_revoke_token"
+    when "revoke_token"
-    else
+      session = env.params.query["session"]
-      return env.redirect referer
-    end
-
-    session = env.params.query["session"]?
-    session ||= ""
-
-    case action
-    when .starts_with? "action_revoke_token"
       Invidious::Database::SessionIDs.delete(sid: session, email: user.email)
     else
       return error_json(400, "Unsupported action #{action}")
@@ -349,40 +341,4 @@ module Invidious::Routes::Account
       return "{}"
     end
   end
-
-  # -------------------
-  #  poToken and visitorData tokens generation
-  # -------------------
-
-  # Generates a poToken & visitorData for the user, server side
-  def generate_tokens(env)
-    locale = env.get("preferences").as(Preferences).locale
-    preferences = env.get("preferences").as(Preferences)
-
-    user = env.get? "user"
-    sid = env.get? "sid"
-    referer = get_referer(env)
-
-    if !user
-      return env.redirect referer
-    end
-
-    user = user.as(User)
-
-    po_token, visitor_data = Tokens.generate_tokens(user.email)
-
-    if po_token.nil? || visitor_data.nil?
-      return error_template(500, "Internal server error. Please submit an issue here IF THE ISSUE PERSISTS: https://git.nadeko.net/Fijxu/invidious/issues")
-    end
-
-    user.preferences.po_token = po_token
-    user.preferences.visitor_data = visitor_data
-
-    Invidious::Database::Users.update_preferences(user)
-
-    REDIS_DB.del("invidious:#{user.email}:po_token")
-    REDIS_DB.del("invidious:#{user.email}:visitor_data")
-
-    templated "user/tokens"
-  end
 end

src/invidious/routes/api/manifest.cr

@@ -8,6 +8,11 @@ module Invidious::Routes::API::Manifest
     id = env.params.url["id"]
     region = env.params.query["region"]?
 
+    if CONFIG.invidious_companion.present?
+      invidious_companion = CONFIG.invidious_companion.sample
+      return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}"
+    end
+
     # Since some implementations create playlists based on resolution regardless of different codecs,
     # we can opt to only add a source to a representation if it has a unique height within that representation
     unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
@@ -55,10 +60,6 @@ module Invidious::Routes::API::Manifest
       end
     end
 
-    audio_streams.reject! do |z|
-      z if z.dig?("audioTrack", "audioIsDefault") == false
-    end
-
     manifest = XML.build(indent: "  ", encoding: "UTF-8") do |xml|
       xml.element("MPD", "xmlns": "urn:mpeg:dash:schema:mpd:2011",
         "profiles": "urn:mpeg:dash:profile:full:2011", minBufferTime: "PT1.5S", type: "static",
@@ -74,17 +75,23 @@ module Invidious::Routes::API::Manifest
               # OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415)
               next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange"))
 
+              audio_track = fmt["audioTrack"]?.try &.as_h? || {} of String => JSON::Any
+              lang = audio_track["id"]?.try &.as_s.split('.')[0] || "und"
+              is_default = audio_track.has_key?("audioIsDefault") ? audio_track["audioIsDefault"].as_bool : i == 0
+              displayname = audio_track["displayName"]?.try &.as_s || "Unknown"
+              bitrate = fmt["bitrate"]
+
               # Different representations of the same audio should be groupped into one AdaptationSet.
               # However, most players don't support auto quality switching, so we have to trick them
               # into providing a quality selector.
               # See https://github.com/iv-org/invidious/issues/3074 for more details.
-              xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: fmt["bitrate"].to_s + "k") do
+              xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: "#{displayname} [#{bitrate}k]", lang: lang) do
                 codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
                 bandwidth = fmt["bitrate"].as_i
                 itag = fmt["itag"].as_i
                 url = fmt["url"].as_s
 
-                xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: i == 0 ? "main" : "alternate")
+                xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: is_default ? "main" : "alternate")
 
                 xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do
                   xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011",
@@ -181,7 +188,7 @@ module Invidious::Routes::API::Manifest
     manifest = response.body
 
     if local
-      manifest = manifest.gsub(/^https:\/\/[^\n"]*/m) do |match|
+      manifest = manifest.gsub(/https:\/\/[^\n"]*/m) do |match|
         uri = URI.parse(match)
         path = uri.path
 
@@ -216,10 +223,16 @@ module Invidious::Routes::API::Manifest
 
         proxy = Invidious::HttpServer::Utils.get_external_proxy
 
+        if CONFIG.https_only
+          scheme = "https://"
+        else
+          scheme = "http://"
+        end
+
         if !proxy.empty?
           "#{proxy}/videoplayback?#{raw_params}"
         else
-          "#{HOST_URL}/videoplayback?#{raw_params}"
+          "#{scheme}#{env.request.headers["Host"]}/videoplayback?#{raw_params}"
         end
       end
     end
@@ -243,7 +256,12 @@ module Invidious::Routes::API::Manifest
     manifest = response.body
 
     if local
-      manifest = manifest.gsub("https://www.youtube.com", HOST_URL)
+      if CONFIG.https_only
+        scheme = "https://"
+      else
+        scheme = "http://"
+      end
+      manifest = manifest.gsub("https://www.youtube.com", "#{scheme}#{env.request.headers["Host"]}")
       manifest = manifest.gsub("index.m3u8", "index.m3u8?local=true")
     end
 

src/invidious/routes/api/v1/authenticated.cr

@@ -226,7 +226,7 @@ module Invidious::Routes::API::V1::Authenticated
     end
 
     playlist = create_playlist(title, privacy, user)
-    env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}"
+    env.response.headers["Location"] = "#{env.request.headers["Host"]}/api/v1/auth/playlists/#{playlist.id}"
     env.response.status_code = 201
     {
       "title"      => title,
@@ -482,7 +482,7 @@ module Invidious::Routes::API::V1::Authenticated
     env.response.content_type = "text/event-stream"
 
     raw_topics = env.params.body["topics"]? || env.params.query["topics"]?
-    topics = raw_topics.try &.split(",").uniq.first(1000)
+    topics = raw_topics.try &.split(",").uniq!.first(1000)
     topics ||= [] of String
 
     create_notification_stream(env, topics, CONNECTION_CHANNEL)

src/invidious/routes/api/v1/channels.cr

@@ -197,6 +197,7 @@ module Invidious::Routes::API::V1::Channels
     get_channel()
 
     # Retrieve continuation from URL parameters
+    sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
     continuation = env.params.query["continuation"]?
 
     if channel.is_age_gated
@@ -211,7 +212,7 @@ module Invidious::Routes::API::V1::Channels
     else
       begin
         videos, next_continuation = Channel::Tabs.get_shorts(
-          channel, continuation: continuation
+          channel, continuation: continuation, sort_by: sort_by
         )
       rescue ex
         return error_json(500, ex)

src/invidious/routes/api/v1/misc.cr

@@ -42,6 +42,9 @@ module Invidious::Routes::API::V1::Misc
     format = env.params.query["format"]?
     format ||= "json"
 
+    listen_param = env.params.query["listen"]?
+    listen = (listen_param == "true" || listen_param == "1")
+
     if plid.starts_with? "RD"
       return env.redirect "/api/v1/mixes/#{plid}"
     end
@@ -85,7 +88,7 @@ module Invidious::Routes::API::V1::Misc
     end
 
     if format == "html"
-      playlist_html = template_playlist(json_response)
+      playlist_html = template_playlist(json_response, listen)
       index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil}
 
       response = {
@@ -111,6 +114,9 @@ module Invidious::Routes::API::V1::Misc
     format = env.params.query["format"]?
     format ||= "json"
 
+    listen_param = env.params.query["listen"]?
+    listen = (listen_param == "true" || listen_param == "1")
+
     begin
       mix = fetch_mix(rdid, continuation, locale: locale)
 
@@ -141,10 +147,8 @@ module Invidious::Routes::API::V1::Misc
                 json.field "authorUrl", "/channel/#{video.ucid}"
 
                 json.field "videoThumbnails" do
-                  json.array do
                   Invidious::JSONify::APIv1.thumbnails(json, video.id)
                 end
-                end
 
                 json.field "index", video.index
                 json.field "lengthSeconds", video.length_seconds
@@ -157,7 +161,7 @@ module Invidious::Routes::API::V1::Misc
 
     if format == "html"
       response = JSON.parse(response)
-      playlist_html = template_mix(response)
+      playlist_html = template_mix(response, listen)
       next_video = response["videos"].as_a.select { |video| !video["author"].as_s.empty? }[0]?.try &.["videoId"]
 
       response = {

src/invidious/routes/api/v1/search.cr

@@ -31,9 +31,7 @@ module Invidious::Routes::API::V1::Search
     query = env.params.query["q"]? || ""
 
     begin
-      client = HTTP::Client.new("suggestqueries-clients6.youtube.com")
+      client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true)
-      client.before_request { |r| add_yt_headers(r) }
-
       url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
 
       response = client.get(url).body

src/invidious/routes/api/v1/videos.cr

@@ -263,61 +263,60 @@ module Invidious::Routes::API::V1::Videos
 
     annotations = ""
 
-    # case source
+    case source
-    # when "archive"
+    when "archive"
-    #   if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id))
+      if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id))
-    #     annotations = cached_annotation.annotations
+        annotations = cached_annotation.annotations
-    #   else
+      else
-    #     index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0')
+        index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0')
 
-    #     # IA doesn't handle leading hyphens,
+        # IA doesn't handle leading hyphens,
-    #     # so we use https://archive.org/details/youtubeannotations_64
+        # so we use https://archive.org/details/youtubeannotations_64
-    #     if index == "62"
+        if index == "62"
-    #       index = "64"
+          index = "64"
-    #       id = id.sub(/^-/, 'A')
+          id = id.sub(/^-/, 'A')
-    #     end
+        end
 
-    #     file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml")
+        file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml")
 
-    #     location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}"))
+        location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}"))
 
-    #     if !location.headers["Location"]?
+        if !location.headers["Location"]?
-    #       env.response.status_code = location.status_code
+          env.response.status_code = location.status_code
-    #     end
+        end
 
-    #     response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"]))
+        response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"]))
 
-    #     if response.body.empty?
+        if response.body.empty?
-    #       haltf env, 404
+          haltf env, 404
-    #     end
+        end
 
-    #     if response.status_code != 200
+        if response.status_code != 200
-    #       haltf env, response.status_code
+          haltf env, response.status_code
-    #     end
+        end
 
-    #     annotations = response.body
+        annotations = response.body
 
-    #     cache_annotation(id, annotations)
+        cache_annotation(id, annotations)
-    #   end
+      end
-    # else # "youtube"
+    else # "youtube"
-    #   response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")
+      response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")
 
-    #   if response.status_code != 200
+      if response.status_code != 200
-    #     haltf env, response.status_code
+        haltf env, response.status_code
-    #   end
+      end
 
-    #   annotations = response.body
+      annotations = response.body
-    # end
+    end
 
-    # etag = sha256(annotations)[0, 16]
+    etag = sha256(annotations)[0, 16]
-    # if env.request.headers["If-None-Match"]?.try &.== etag
+    if env.request.headers["If-None-Match"]?.try &.== etag
-    #   haltf env, 304
+      haltf env, 304
-    # else
+    else
-    #   env.response.headers["ETag"] = etag
+      env.response.headers["ETag"] = etag
-    #   annotations
-    # end
       annotations
     end
+  end
 
   def self.comments(env)
     locale = env.get("preferences").as(Preferences).locale

src/invidious/routes/before_all.cr

@@ -20,12 +20,25 @@ module Invidious::Routes::BeforeAll
     env.response.headers["X-XSS-Protection"] = "1; mode=block"
     env.response.headers["X-Content-Type-Options"] = "nosniff"
 
+    extra_media_csp = ""
+    extra_connect_csp = ""
+
+    if CONFIG.invidious_companion.present?
+      extra_media_csp = " #{CONFIG.invidious_companion.sample.public_url}"
+      extra_connect_csp = " #{CONFIG.invidious_companion.sample.public_url}"
+    end
+
+    if !CONFIG.external_videoplayback_proxy.empty?
+      CONFIG.external_videoplayback_proxy.each do |proxy|
+        extra_media_csp += " #{proxy}"
+        extra_connect_csp += " #{proxy}"
+      end
+    end
+
     # Allow media resources to be loaded from google servers
     # TODO: check if *.youtube.com can be removed
     if CONFIG.disabled?("local") || !preferences.local
-      extra_media_csp = " https://*.googlevideo.com:443 https://*.youtube.com:443"
+      extra_media_csp += " https://*.googlevideo.com:443 https://*.youtube.com:443"
-    else
-      extra_media_csp = ""
     end
 
     # Only allow the pages at /embed/* to be embedded
@@ -43,9 +56,9 @@ module Invidious::Routes::BeforeAll
       "style-src 'self' 'unsafe-inline'",
       "img-src 'self' data:",
       "font-src 'self' data:",
-      "connect-src 'self'" + EXT_VIDEOP_LIST,
+      "connect-src 'self'" + extra_connect_csp,
       "manifest-src 'self'",
-      "media-src 'self' blob:" + extra_media_csp + EXT_VIDEOP_LIST,
+      "media-src 'self' blob:" + extra_media_csp,
       "child-src 'self' blob:",
       "frame-src 'self'",
       "frame-ancestors " + frame_ancestors,

src/invidious/routes/channels.cr

@@ -20,10 +20,11 @@ module Invidious::Routes::Channels
     sort_by = env.params.query["sort_by"]?.try &.downcase
 
     if channel.auto_generated
+      sort_by ||= "last"
       sort_options = {"last", "oldest", "newest"}
 
       items, next_continuation = fetch_channel_playlists(
-        channel.ucid, channel.author, continuation, (sort_by || "last")
+        channel.ucid, channel.author, continuation, sort_by
       )
 
       items.uniq! do |item|
@@ -49,9 +50,11 @@ module Invidious::Routes::Channels
         end
         next_continuation = nil
       else
+        sort_by ||= "newest"
         sort_options = {"newest", "oldest", "popular"}
-        items, next_continuation = Channel::Tabs.get_videos(
+
-          channel, continuation: continuation, sort_by: (sort_by || "newest")
+        items, next_continuation = Channel::Tabs.get_60_videos(
+          channel, continuation: continuation, sort_by: sort_by
         )
       end
     end
@@ -82,13 +85,12 @@ module Invidious::Routes::Channels
       end
       next_continuation = nil
     else
-      # TODO: support sort option for shorts
+      sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
-      sort_by = ""
+      sort_options = {"newest", "oldest", "popular"}
-      sort_options = [] of String
 
       # Fetch items and continuation token
       items, next_continuation = Channel::Tabs.get_shorts(
-        channel, continuation: continuation
+        channel, continuation: continuation, sort_by: sort_by
       )
     end
 

src/invidious/routes/feeds.cr

@@ -194,6 +194,7 @@ module Invidious::Routes::Feeds
         length_seconds:     0,
         premiere_timestamp: nil,
         author_verified:    false,
+        author_thumbnail:   nil,
         badges:             VideoBadges::None,
       })
     end
@@ -416,18 +417,22 @@ module Invidious::Routes::Feeds
         author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
         published = Time.parse_rfc3339(entry.xpath_node("default:published", namespaces).not_nil!.content)
         updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
+        ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
+        title = entry.xpath_node("default:title", namespaces).not_nil!.content
 
+        if CONFIG.use_innertube_for_feeds
           begin
-          video = get_video(id, force_refresh: true)
+            video_ = get_video(id, force_refresh: true)
           rescue
             next # skip this video since it raised an exception (e.g. it is a scheduled live event)
           end
+        end
 
         if CONFIG.enable_user_notifications
           # Deliver notifications to `/api/v1/auth/notifications`
           payload = {
-            "topic"     => video.ucid,
+            "topic"     => ucid,
-            "videoId"   => video.id,
+            "videoId"   => id,
             "published" => published.to_unix,
           }.to_json
           PG_DB.exec("NOTIFY notifications, E'#{payload}'")
@@ -435,15 +440,15 @@ module Invidious::Routes::Feeds
 
         video = ChannelVideo.new({
           id:                 id,
-          title:              video.title,
+          title:              title,
           published:          published,
           updated:            updated,
-          ucid:               video.ucid,
+          ucid:               ucid,
           author:             author,
-          length_seconds:     video.length_seconds,
+          length_seconds:     video_.try &.length_seconds || 0,
-          live_now:           video.live_now,
+          live_now:           video_.try &.live_now || false,
-          premiere_timestamp: video.premiere_timestamp,
+          premiere_timestamp: video_.try &.premiere_timestamp || nil,
-          views:              video.views,
+          views:              video_.try &.views || nil,
         })
 
         was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)

src/invidious/routes/login.cr

@@ -64,6 +64,8 @@ module Invidious::Routes::Login
           # TOR or I2P address
           if alt = CONFIG.alternative_domains.index(env.request.headers["Host"])
             env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.alternative_domains[alt], sid)
+          elsif alt = CONFIG.backend_domains.index(env.request.headers["Host"])
+            env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.backend_domains[alt], sid)
           else
             env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid)
           end
@@ -170,6 +172,8 @@ module Invidious::Routes::Login
         # TOR or I2P address
         if alt = CONFIG.alternative_domains.index(env.request.headers["Host"])
           env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.alternative_domains[alt], sid)
+        elsif alt = CONFIG.backend_domains.index(env.request.headers["Host"])
+          env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.backend_domains[alt], sid)
         else
           env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid)
         end

src/invidious/routes/playlists.cr

@@ -304,23 +304,6 @@ module Invidious::Routes::Playlists
       end
     end
 
-    if env.params.query["action_create_playlist"]?
-      action = "action_create_playlist"
-    elsif env.params.query["action_delete_playlist"]?
-      action = "action_delete_playlist"
-    elsif env.params.query["action_edit_playlist"]?
-      action = "action_edit_playlist"
-    elsif env.params.query["action_add_video"]?
-      action = "action_add_video"
-      video_id = env.params.query["video_id"]
-    elsif env.params.query["action_remove_video"]?
-      action = "action_remove_video"
-    elsif env.params.query["action_move_video_before"]?
-      action = "action_move_video_before"
-    else
-      return env.redirect referer
-    end
-
     begin
       playlist_id = env.params.query["playlist_id"]
       playlist = get_playlist(playlist_id).as(InvidiousPlaylist)
@@ -335,12 +318,8 @@ module Invidious::Routes::Playlists
       end
     end
 
-    email = user.email
+    case action = env.params.query["action"]?
-
+    when "add_video"
-    case action
-    when "action_edit_playlist"
-      # TODO: Playlist stub
-    when "action_add_video"
       if playlist.index.size >= CONFIG.playlist_length_limit
         if redirect
           return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
@@ -377,12 +356,14 @@ module Invidious::Routes::Playlists
 
       Invidious::Database::PlaylistVideos.insert(playlist_video)
       Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index)
-    when "action_remove_video"
+    when "remove_video"
       index = env.params.query["set_video_id"]
       Invidious::Database::PlaylistVideos.delete(index)
       Invidious::Database::Playlists.update_video_removed(playlist_id, index)
-    when "action_move_video_before"
+    when "move_video_before"
       # TODO: Playlist stub
+    when nil
+      return error_json(400, "Missing action")
     else
       return error_json(400, "Unsupported action #{action}")
     end

src/invidious/routes/preferences.cr

@@ -86,12 +86,6 @@ module Invidious::Routes::PreferencesRoute
     show_nick ||= "off"
     show_nick = show_nick == "on"
 
-    po_token = env.params.body["po_token"]?.try &.as(String)
-    po_token ||= CONFIG.default_user_preferences.po_token
-
-    visitor_data = env.params.body["visitor_data"]?.try &.as(String)
-    visitor_data ||= CONFIG.default_user_preferences.visitor_data
-
     comments = [] of String
     2.times do |i|
       comments << (env.params.body["comments[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.comments[i])
@@ -186,8 +180,6 @@ module Invidious::Routes::PreferencesRoute
       vr_mode:                     vr_mode,
       show_nick:                   show_nick,
       save_player_pos:             save_player_pos,
-      po_token:                    po_token,
-      visitor_data:                visitor_data,
     }.to_json)
 
     if user = env.get? "user"
@@ -236,6 +228,8 @@ module Invidious::Routes::PreferencesRoute
       # TOR or I2P address
       if alt = CONFIG.alternative_domains.index(env.request.headers["Host"])
         env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.alternative_domains[alt], preferences)
+      elsif alt = CONFIG.backend_domains.index(env.request.headers["Host"])
+        env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.backend_domains[alt], preferences)
       else
         env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences)
       end
@@ -277,6 +271,8 @@ module Invidious::Routes::PreferencesRoute
       # TOR or I2P address
       if alt = CONFIG.alternative_domains.index(env.request.headers["Host"])
         env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.alternative_domains[alt], preferences)
+      elsif alt = CONFIG.backend_domains.index(env.request.headers["Host"])
+        env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.backend_domains[alt], preferences)
       else
         env.response.cookies["PREFS"] = Invidious::User::Cookies.prefs(CONFIG.domain, preferences)
       end

src/invidious/routes/subscriptions.cr

@@ -32,24 +32,16 @@ module Invidious::Routes::Subscriptions
       end
     end
 
-    if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1
-      action = "action_create_subscription_to_channel"
-    elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1
-      action = "action_remove_subscriptions"
-    else
-      return env.redirect referer
-    end
-
     channel_id = env.params.query["c"]?
     channel_id ||= ""
 
-    case action
+    case action = env.params.query["action"]?
-    when "action_create_subscription_to_channel"
+    when "create_subscription_to_channel"
       if !user.subscriptions.includes? channel_id
         get_channel(channel_id)
         Invidious::Database::Users.subscribe_channel(user, channel_id)
       end
-    when "action_remove_subscriptions"
+    when "remove_subscriptions"
       Invidious::Database::Users.unsubscribe_channel(user, channel_id)
     else
       return error_json(400, "Unsupported action #{action}")

src/invidious/routes/switch_backend.cr

@@ -1,18 +0,0 @@
-{% skip_file if flag?(:api_only) %}
-
-module Invidious::Routes::BackendSwitcher
-  def self.switch(env)
-    referer = get_referer(env)
-    backend_id = env.params.query["backend_id"]
-
-    # Checks if there is any alternative domain, like a second domain name,
-    # TOR or I2P address
-    if alt = CONFIG.alternative_domains.index(env.request.headers["Host"])
-      env.response.cookies["SERVER_ID"] = Invidious::User::Cookies.server_id(CONFIG.alternative_domains[alt], backend_id)
-    else
-      env.response.cookies["SERVER_ID"] = Invidious::User::Cookies.server_id(CONFIG.domain, backend_id)
-    end
-
-    env.redirect referer
-  end
-end

src/invidious/routes/video_playback.cr

@@ -47,11 +47,7 @@ module Invidious::Routes::VideoPlayback
       headers["Range"] = "bytes=#{range_for_head}"
     end
 
-    headers["Origin"] = "https://www.youtube.com"
+    client = make_client(URI.parse(host), region, force_resolve: true)
-    headers["Referer"] = "https://www.youtube.com/"
-    headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
-
-    client = make_client(URI.parse(host), region, force_resolve = true)
     response = HTTP::Client::Response.new(500)
     error = ""
     5.times do
@@ -66,7 +62,7 @@ module Invidious::Routes::VideoPlayback
           if new_host != host
             host = new_host
             client.close
-            client = make_client(URI.parse(new_host), region, force_resolve = true)
+            client = make_client(URI.parse(new_host), region, force_resolve: true)
           end
 
           url = "#{location.request_target}&host=#{location.host}#{region ? "&region=#{region}" : ""}"
@@ -80,7 +76,7 @@ module Invidious::Routes::VideoPlayback
         fvip = "3"
 
         host = "https://r#{fvip}---#{mn}.googlevideo.com"
-        client = make_client(URI.parse(host), region, force_resolve = true)
+        client = make_client(URI.parse(host), region, force_resolve: true)
       rescue ex
         error = ex.message
       end
@@ -173,10 +169,13 @@ module Invidious::Routes::VideoPlayback
               env.response.headers["Access-Control-Allow-Origin"] = "*"
 
               if location = resp.headers["Location"]?
-                location = URI.parse(location)
+                url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
-                location = "#{location.request_target}&host=#{location.host}#{region ? "&region=#{region}" : ""}"
 
-                env.redirect location
+                if title = query_params["title"]?
+                  url = "#{url}&title=#{URI.encode_www_form(title)}"
+                end
+
+                env.redirect url
                 break
               end
 
@@ -205,7 +204,7 @@ module Invidious::Routes::VideoPlayback
             break
           else
             client.close
-            client = make_client(URI.parse(host), region, force_resolve = true)
+            client = make_client(URI.parse(host), region, force_resolve: true)
           end
         end
 
@@ -262,6 +261,11 @@ module Invidious::Routes::VideoPlayback
   # YouTube /videoplayback links expire after 6 hours,
   # so we have a mechanism here to redirect to the latest version
   def self.latest_version(env)
+    if CONFIG.invidious_companion.present?
+      invidious_companion = CONFIG.invidious_companion.sample
+      return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}"
+    end
+
     id = env.params.query["id"]?
     itag = env.params.query["itag"]?.try &.to_i?
 

src/invidious/routes/watch.cr

@@ -3,14 +3,6 @@
 module Invidious::Routes::Watch
   def self.handle(env)
     locale = env.get("preferences").as(Preferences).locale
-    if !CONFIG.ignore_user_tokens
-      user_po_token = env.get("preferences").as(Preferences).po_token
-      user_visitor_data = env.get("preferences").as(Preferences).visitor_data
-    else
-      user_po_token = ""
-      user_visitor_data = ""
-    end
-
     region = env.params.query["region"]?
 
     if env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+")
@@ -60,7 +52,7 @@ module Invidious::Routes::Watch
     env.params.query.delete_all("listen")
 
     begin
-      video = get_video(id, region: params.region, po_token: user_po_token, visitor_data: user_visitor_data)
+      video = get_video(id, region: params.region)
     rescue ex : NotFoundException
       LOGGER.error("get_video not found: #{id} : #{ex.message}")
       return error_template(404, ex)
@@ -139,9 +131,6 @@ module Invidious::Routes::Watch
       fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
     end
 
-    # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
-    adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
-
     video_streams = video.video_streams
     audio_streams = video.audio_streams
 
@@ -152,11 +141,6 @@ module Invidious::Routes::Watch
       end
     end
 
-	# Removes non default audio tracks
-    audio_streams.reject! do |z|
-      z if z.dig?("audioTrack", "audioIsDefault") == false
-    end
-
     # Older videos may not have audio sources available.
     # We redirect here so they're not unplayable
     if audio_streams.empty? && !video.live_now
@@ -276,18 +260,10 @@ module Invidious::Routes::Watch
       end
     end
 
-    if env.params.query["action_mark_watched"]?
+    case action = env.params.query["action"]?
-      action = "action_mark_watched"
+    when "mark_watched"
-    elsif env.params.query["action_mark_unwatched"]?
-      action = "action_mark_unwatched"
-    else
-      return env.redirect referer
-    end
-
-    case action
-    when "action_mark_watched"
       Invidious::Database::Users.mark_watched(user, id)
-    when "action_mark_unwatched"
+    when "mark_unwatched"
       Invidious::Database::Users.mark_unwatched(user, id)
     else
       return error_json(400, "Unsupported action #{action}")
@@ -355,14 +331,18 @@ module Invidious::Routes::Watch
       env.params.query["label"] = URI.decode_www_form(label.as_s)
 
       return Invidious::Routes::API::V1::Videos.captions(env)
-    elsif itag = download_widget["itag"]?.try &.as_i
+    elsif itag = download_widget["itag"]?.try &.as_i.to_s
       # URL params specific to /latest_version
       env.params.query["id"] = video_id
-      env.params.query["itag"] = itag.to_s
       env.params.query["title"] = filename
       env.params.query["local"] = "true"
 
+      if (CONFIG.invidious_companion.present?)
+        video = get_video(video_id)
+        return env.redirect "#{video.invidious_companion["baseUrl"].as_s}/latest_version?#{env.params.query}"
+      else
         return Invidious::Routes::VideoPlayback.latest_version(env)
+      end
     else
       return error_template(400, "Invalid label or itag")
     end

src/invidious/routing.cr

@@ -21,7 +21,6 @@ module Invidious::Routing
       get "/privacy", Routes::Misc, :privacy
       get "/licenses", Routes::Misc, :licenses
       get "/redirect", Routes::Misc, :cross_instance_redirect
-      get "/switchbackend", Routes::BackendSwitcher, :switch
 
       self.register_channel_routes
       self.register_watch_routes
@@ -77,7 +76,6 @@ module Invidious::Routing
     post "/authorize_token", Routes::Account, :post_authorize_token
     get "/token_manager", Routes::Account, :token_manager
     post "/token_ajax", Routes::Account, :token_ajax
-    get "/generate_tokens", Routes::Account, :generate_tokens
     post "/subscription_ajax", Routes::Subscriptions, :toggle_subscription
     get "/subscription_manager", Routes::Subscriptions, :subscription_manager
   end
@@ -245,17 +243,16 @@ module Invidious::Routing
 
       # Channels
       get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
+      get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest
+      get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos
       get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
       get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
       get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts
       get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases
-
+      get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists
+      get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community
       get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
-
+      get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search
-      {% for route in {"videos", "latest", "playlists", "community", "search"} %}
-        get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
-        get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}
-      {% end %}
 
       # Posts
       get "/api/v1/post/:id", {{namespace}}::Channels, :post
@@ -273,11 +270,6 @@ module Invidious::Routing
 
       # Authenticated
 
-      # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr
-      #
-      # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
-      # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
-
       get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences
       post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences
 

src/invidious/search/filters.cr

@@ -75,7 +75,7 @@ module Invidious::Search
       @type : Type = Type::All,
       @duration : Duration = Duration::None,
       @features : Features = Features::None,
-      @sort : Sort = Sort::Relevance
+      @sort : Sort = Sort::Relevance,
     )
     end
 

src/invidious/search/query.cr

@@ -47,7 +47,7 @@ module Invidious::Search
     def initialize(
       params : HTTP::Params,
       @type : Type = Type::Regular,
-      @region : String? = nil
+      @region : String? = nil,
     )
       # Get the raw search query string (common to all search types). In
       # Regular search mode, also look for the `search_query` URL parameter

src/invidious/user/cookies.cr

@@ -45,18 +45,5 @@ struct Invidious::User
         samesite: HTTP::Cookie::SameSite::Lax
       )
     end
-
-    # Server ID (SERVER_ID) cookie used for Sticky Sessions
-    # Parameter "domain" comes from the global config
-    def server_id(domain : String?, server_id) : HTTP::Cookie
-      return HTTP::Cookie.new(
-        name: "SERVER_ID",
-        domain: domain,
-        value: server_id,
-        secure: false,
-        http_only: true,
-        samesite: HTTP::Cookie::SameSite::Lax
-      )
-    end
   end
 end

src/invidious/user/imports.cr

@@ -290,26 +290,22 @@ struct Invidious::User
     end
 
     def from_newpipe(user : User, body : String) : Bool
-      io = IO::Memory.new(body)
+      Compress::Zip::File.open(IO::Memory.new(body), true) do |file|
-
+        entry = file.entries.find { |file_entry| file_entry.filename == "newpipe.db" }
-      Compress::Zip::File.open(io) do |file|
+        return false if entry.nil?
-        file.entries.each do |entry|
         entry.open do |file_io|
           # Ensure max size of 4MB
           io_sized = IO::Sized.new(file_io, 0x400000)
 
-            next if entry.filename != "newpipe.db"
+          begin
-
+            temp = File.tempfile(".db") do |tempfile|
-            tempfile = File.tempfile(".db")
-
               begin
                 File.write(tempfile.path, io_sized.gets_to_end)
               rescue
                 return false
               end
 
-            db = DB.open("sqlite3://" + tempfile.path)
+              DB.open("sqlite3://" + tempfile.path) do |db|
-
                 user.watched += db.query_all("SELECT url FROM streams", as: String)
                   .map(&.lchop("https://www.youtube.com/watch?v="))
 
@@ -323,9 +319,10 @@ struct Invidious::User
                 user.subscriptions = get_batch_channels(user.subscriptions)
 
                 Invidious::Database::Users.update_subscriptions(user)
-
+              end
-            db.close
+            end
-            tempfile.delete
+          ensure
+            temp.delete if !temp.nil?
           end
         end
       end

src/invidious/user/preferences.cr

@@ -57,10 +57,6 @@ struct Preferences
   property volume : Int32 = CONFIG.default_user_preferences.volume
   property save_player_pos : Bool = CONFIG.default_user_preferences.save_player_pos
 
-  @[YAML::Field(converter: Preferences::ProcessString)]
-  property po_token : String = ""
-  property visitor_data : String = ""
-
   module BoolToString
     def self.to_json(value : String, json : JSON::Builder)
       json.string value

src/invidious/videos.cr

@@ -15,7 +15,7 @@ struct Video
   # NOTE: don't forget to bump this number if any change is made to
   # the `params` structure in videos/parser.cr!!!
   #
-  SCHEMA_VERSION = 2
+  SCHEMA_VERSION = 3
 
   property id : String
 
@@ -106,7 +106,7 @@ struct Video
     if formats = info.dig?("streamingData", "adaptiveFormats")
       return formats
         .as_a.map(&.as_h)
-        .sort_by! { |f| f["width"]?.try &.as_i || 0 }
+        .sort_by! { |f| f["width"]?.try &.as_i || f["audioTrack"]?.try { |a| a["audioIsDefault"]?.try { |v| v.as_bool ? -1 : 0 } } || 0 }
     else
       return [] of Hash(String, JSON::Any)
     end
@@ -192,6 +192,10 @@ struct Video
     }
   end
 
+  def invidious_companion : Hash(String, JSON::Any)?
+    info["invidiousCompanion"]?.try &.as_h || {} of String => JSON::Any
+  end
+
   # Macros defining getters/setters for various types of data
 
   private macro getset_string(name)
@@ -294,7 +298,7 @@ struct Video
   predicate_bool upcoming, isUpcoming
 end
 
-def get_video(id, refresh = true, region = nil, force_refresh = false, po_token = "", visitor_data = "")
+def get_video(id, refresh = true, region = nil, force_refresh = false)
   if (video = Invidious::Database::Videos.select(id)) && !region
     # If record was last updated over 10 minutes ago, or video has since premiered,
     # refresh (expire param in response lasts for 6 hours)
@@ -304,7 +308,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false, po_token
        force_refresh ||
        video.schema_version != Video::SCHEMA_VERSION # cache control
       begin
-        video = fetch_video(id, region, po_token, visitor_data)
+        video = fetch_video(id, region)
         Invidious::Database::Videos.insert(video)
       rescue ex
         Invidious::Database::Videos.delete(id)
@@ -312,7 +316,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false, po_token
       end
     end
   else
-    video = fetch_video(id, region, po_token, visitor_data)
+    video = fetch_video(id, region)
     Invidious::Database::Videos.insert(video) if !region
   end
 
@@ -320,11 +324,11 @@ def get_video(id, refresh = true, region = nil, force_refresh = false, po_token
 rescue DB::Error
   # Avoid common `DB::PoolRetryAttemptsExceeded` error and friends
   # Note: All DB errors inherit from `DB::Error`
-  return fetch_video(id, region, po_token, visitor_data)
+  return fetch_video(id, region)
 end
 
-def fetch_video(id, region, po_token, visitor_data)
+def fetch_video(id, region)
-  info = extract_video_info(video_id: id, user_po_token: po_token, user_visitor_data: visitor_data)
+  info = extract_video_info(video_id: id)
 
   if reason = info["reason"]?
     if reason == "Video unavailable"

src/invidious/videos/parser.cr

@@ -36,6 +36,13 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
 
   LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container")
 
+  if published_time_text = related["publishedTimeText"]?
+    decoded_time = decode_date(published_time_text["simpleText"].to_s)
+    published = decoded_time.to_rfc3339.to_s
+  else
+    published = nil
+  end
+
   # TODO: when refactoring video types, make a struct for related videos
   # or reuse an existing type, if that fits.
   return {
@@ -47,20 +54,16 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
     "view_count"       => JSON::Any.new(view_count || "0"),
     "short_view_count" => JSON::Any.new(short_view_count || "0"),
     "author_verified"  => JSON::Any.new(author_verified),
+    "published"        => JSON::Any.new(published || ""),
   }
 end
 
-def extract_video_info(video_id : String, user_po_token, user_visitor_data)
+def extract_video_info(video_id : String)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
 
-  redis_po_token, redis_visitor_data = Tokens.get_tokens
-
-  po_token = (user_po_token if !user_po_token.empty?) || redis_po_token || CONFIG.po_token
-  visitor_data = (user_visitor_data if !user_visitor_data.empty?) || redis_visitor_data || CONFIG.visitor_data
-
   # Fetch data from the player endpoint
-  player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config, po_token: po_token, visitor_data: visitor_data)
+  player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
 
   playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
 
@@ -105,26 +108,18 @@ def extract_video_info(video_id : String, user_po_token, user_visitor_data)
   params = parse_video_info(video_id, player_response)
   params["reason"] = JSON::Any.new(reason) if reason
 
+  if CONFIG.invidious_companion.present?
     new_player_response = nil
 
-  # Don't use Android client if po_token is passed because po_token doesn't
+    # Don't use Android test suite client if po_token is passed because po_token doesn't
-  # work for Android client.
+    # work for Android test suite client.
-  if reason.nil? && po_token.nil?
+    if reason.nil? && CONFIG.po_token.nil?
       # Fetch the video streams using an Android client in order to get the
       # decrypted URLs and maybe fix throttling issues (#2194). See the
       # following issue for an explanation about decrypted URLs:
       # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
       client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
-    new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data)
+      new_player_response = try_fetch_streaming_data(video_id, client_config)
-  end
-
-  # Last hope
-  # Only trigger if reason found and po_token or didn't work wth Android client.
-  # TvHtml5ScreenEmbed now requires sig helper for it to work but po_token is not required
-  # if the IP address is not blocked.
-  if po_token.nil? && reason || po_token.nil? && new_player_response.nil?
-    client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
-    new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data)
     end
 
     # Replace player response and reset reason
@@ -136,8 +131,9 @@ def extract_video_info(video_id : String, user_po_token, user_visitor_data)
       player_response = new_player_response
       params.delete("reason")
     end
+  end
 
-  {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
+  {"captions", "playabilityStatus", "playerConfig", "storyboards", "invidiousCompanion"}.each do |f|
     params[f] = player_response[f] if player_response[f]?
   end
 
@@ -145,7 +141,7 @@ def extract_video_info(video_id : String, user_po_token, user_visitor_data)
   if streaming_data = player_response["streamingData"]?
     %w[formats adaptiveFormats].each do |key|
       streaming_data.as_h[key]?.try &.as_a.each do |format|
-        format.as_h["url"] = JSON::Any.new(convert_url(format, po_token))
+        format.as_h["url"] = JSON::Any.new(convert_url(format))
       end
     end
 
@@ -158,9 +154,9 @@ def extract_video_info(video_id : String, user_po_token, user_visitor_data)
   return params
 end
 
-def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig, po_token, visitor_data) : Hash(String, JSON::Any)?
+def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
   LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
-  response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config, po_token: po_token, visitor_data: visitor_data)
+  response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)
 
   playability_status = response["playabilityStatus"]["status"]
   LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
@@ -229,8 +225,17 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
     .try { |t| Time.parse_rfc3339(t.as_s) }
 
+  premiere_timestamp ||= player_response.dig?(
+    "playabilityStatus", "liveStreamability",
+    "liveStreamabilityRenderer", "offlineSlate",
+    "liveStreamOfflineSlateRenderer", "scheduledStartTime"
+  )
+    .try &.as_s.to_i64
+      .try { |t| Time.unix(t) }
+
   live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow")
-    .try &.as_bool || false
+    .try &.as_bool
+  live_now ||= video_details.dig?("isLive").try &.as_bool || false
 
   post_live_dvr = video_details.dig?("isPostLiveDvr")
     .try &.as_bool || false
@@ -460,7 +465,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   return params
 end
 
-private def convert_url(fmt, po_token)
+private def convert_url(fmt)
   if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
     sp = cfr["sp"]
     url = URI.parse(cfr["url"])
@@ -478,9 +483,7 @@ private def convert_url(fmt, po_token)
   n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
   params["n"] = n if n
 
-  if !po_token.nil?
+  if token = CONFIG.po_token
-    params["pot"] = po_token
-  elsif token = CONFIG.po_token
     params["pot"] = token
   end
 

src/invidious/videos/storyboard.cr

@@ -20,7 +20,7 @@ module Invidious::Videos
 
     def initialize(
       *, @url, @width, @height, @count, @interval,
-      @rows, @columns, @images_count
+      @rows, @columns, @images_count,
     )
       authority = /(i\d?).ytimg.com/.match!(@url.host.not_nil!)[1]?
 

src/invidious/views/channel.ecr

@@ -2,6 +2,7 @@
   ucid = channel.ucid
   author = HTML.escape(channel.author)
   channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+  host = env.request.headers["Host"]
 
   relative_url =
     case selected_tab
@@ -28,15 +29,15 @@
 <%- if selected_tab.videos? -%>
 <meta name="description" content="<%= channel.description %>">
 <meta property="og:site_name" content="Invidious">
-<meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>">
+<meta property="og:url" content="<%= host %>/channel/<%= ucid %>">
 <meta property="og:title" content="<%= author %>">
-<meta property="og:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>">
+<meta property="og:image" content="<%= host %>/ggpht<%= channel_profile_pic %>">
 <meta property="og:description" content="<%= channel.description %>">
 <meta name="twitter:card" content="summary">
-<meta name="twitter:url" content="<%= HOST_URL %>/channel/<%= ucid %>">
+<meta name="twitter:url" content="<%= host %>/channel/<%= ucid %>">
 <meta name="twitter:title" content="<%= author %>">
 <meta name="twitter:description" content="<%= channel.description %>">
-<meta name="twitter:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>">
+<meta name="twitter:image" content="<%= host %>/ggpht<%= channel_profile_pic %>">
 <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" />
 <%- end -%>
 

src/invidious/views/components/item.ecr

@@ -103,12 +103,15 @@
               if item.is_a?(PlaylistVideo)
                 link_url = "/watch?v=#{item.id}&list=#{item.plid}&index=#{item.index}"
                 endpoint_params = "?v=#{item.id}&list=#{item.plid}"
+                embed_id = item.id
               elsif item.is_a?(MixVideo)
                 link_url = "/watch?v=#{item.id}&list=#{item.rdid}"
                 endpoint_params = "?v=#{item.id}&list=#{item.rdid}"
+                embed_id = item.id
               else
                 link_url = "/watch?v=#{item.id}"
                 endpoint_params = "?v=#{item.id}"
+                embed_id = item.id
               end
             -%>
 
@@ -128,7 +131,7 @@
 
                 <div class="top-left-overlay">
                     <%- if env.get? "show_watched" -%>
-                        <form data-onsubmit="return_false" action="/watch_ajax?action_mark_watched=1&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post">
+                        <form data-onsubmit="return_false" action="/watch_ajax?action=mark_watched&id=<%= item.id %>&referer=<%= env.get("current_page") %>" method="post">
                             <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>">
                             <button type="submit" class="pure-button pure-button-secondary low-profile"
                                     data-onclick="mark_watched" data-id="<%= item.id %>">
@@ -138,14 +141,14 @@
                     <%- end -%>
 
                     <%- if plid_form = env.get?("add_playlist_items") -%>
-                        <%- form_parameters = "action_add_video=1&video_id=#{item.id}&playlist_id=#{plid_form}&referer=#{env.get("current_page")}" -%>
+                        <%- form_parameters = "action=add_video&video_id=#{item.id}&playlist_id=#{plid_form}&referer=#{env.get("current_page")}" -%>
                         <form data-onsubmit="return_false" action="/playlist_ajax?<%= form_parameters %>" method="post">
                             <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>">
                             <button type="submit" class="pure-button pure-button-secondary low-profile"
                                     data-onclick="add_playlist_item" data-id="<%= item.id %>" data-plid="<%= plid_form %>"><i class="icon ion-md-add"></i></button>
                         </form>
                     <%- elsif item.is_a?(PlaylistVideo) && (plid_form = env.get?("remove_playlist_items")) -%>
-                        <%- form_parameters = "action_remove_video=1&set_video_id=#{item.index}&playlist_id=#{plid_form}&referer=#{env.get("current_page")}" -%>
+                        <%- form_parameters = "action=remove_video&set_video_id=#{item.index}&playlist_id=#{plid_form}&referer=#{env.get("current_page")}" -%>
                         <form data-onsubmit="return_false" action="/playlist_ajax?<%= form_parameters %>" method="post">
                             <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>">
                             <button type="submit" class="pure-button pure-button-secondary low-profile"

src/invidious/views/components/player.ecr

@@ -22,6 +22,8 @@
                audio_streams.each_with_index do |fmt, i|
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
+                src_url = video.invidious_companion["baseUrl"].as_s + src_url + 
+                            "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?)
 
                 bitrate = fmt["bitrate"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)
@@ -34,8 +36,12 @@
                 <% end %>
             <% end %>
         <% else %>
-            <% if params.quality == "dash" %>
+            <% if params.quality == "dash"
-                <source src="/api/manifest/dash/id/<%= video.id %>?local=true&unique_res=1" type='application/dash+xml' label="dash">
+               src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1"
+               src_url = video.invidious_companion["baseUrl"].as_s + src_url +
+                            "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?)
+            %>
+                <source src="<%= src_url %>" type='application/dash+xml' label="dash">
             <% end %>
 
             <%
@@ -44,6 +50,8 @@
             fmt_stream.each_with_index do |fmt, i|
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
+                src_url = video.invidious_companion["baseUrl"].as_s + src_url +
+                            "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?)
 
                 quality = fmt["quality"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)

src/invidious/views/components/subscribe_widget.ecr

@@ -1,13 +1,13 @@
 <% if user %>
     <% if subscriptions.includes? ucid %>
-            <form action="/subscription_ajax?action_remove_subscriptions=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post">
+            <form action="/subscription_ajax?action=remove_subscriptions&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post">
                 <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>">
                 <button data-type="unsubscribe" id="subscribe" class="pure-button pure-button-primary">
                     <b><input style="all:unset" type="submit" value="<%= translate(locale, "Unsubscribe") %> | <%= sub_count_text %>"></b>
                 </button>
             </form>
     <% else %>
-            <form action="/subscription_ajax?action_create_subscription_to_channel=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post">
+            <form action="/subscription_ajax?action=create_subscription_to_channel&c=<%= ucid %>&referer=<%= env.get("current_page") %>" method="post">
                 <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>">
                 <button data-type="subscribe" id="subscribe" class="pure-button pure-button-primary">
                     <b><input style="all:unset" type="submit" value="<%= translate(locale, "Subscribe") %> | <%= sub_count_text %>"></b>

src/invidious/views/components/video-context-buttons.ecr

@@ -1,5 +1,8 @@
 <div class="flex-right flexible">
     <div class="icon-buttons">
+        <a title="<%=translate(locale, "videoinfo_youTube_embed_link")%>" rel="noreferrer noopener" href="https://www.youtube.com/embed/<%=embed_id%>">
+            <i class="icon ion-md-open"></i>
+        </a>
         <a title="<%=translate(locale, "videoinfo_watch_on_youTube")%>" rel="noreferrer noopener" href="https://www.youtube.com/watch<%=endpoint_params%>">
             <i class="icon ion-logo-youtube"></i>
         </a>

src/invidious/views/feeds/history.ecr

@@ -37,7 +37,7 @@
                     </a>
 
                     <div class="top-left-overlay"><div class="watched">
-                    <form data-onsubmit="return_false" action="/watch_ajax?action_mark_unwatched=1&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post">
+                    <form data-onsubmit="return_false" action="/watch_ajax?action=mark_unwatched&id=<%= item %>&referer=<%= env.get("current_page") %>" method="post">
                         <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
                         <button type="submit" class="pure-button pure-button-secondary low-profile"
                                 data-onclick="mark_unwatched" data-id="<%= item %>"><i class="icon ion-md-trash"></i></button>

src/invidious/views/template.ecr

@@ -1,7 +1,8 @@
 <%
 	locale = env.get("preferences").as(Preferences).locale
 	dark_mode = env.get("preferences").as(Preferences).dark_mode
-	current_backend = env.request.cookies["SERVER_ID"]?.try &.value
+	current_backend = env.request.cookies[CONFIG.server_id_cookie_name]?.try &.value || env.request.headers["Host"]
+    current_external_videoplayback_proxy = Invidious::HttpServer::Utils.get_external_proxy()
 %>
 <!DOCTYPE html>
 <html lang="<%= locale %>">
@@ -106,6 +107,7 @@
 				</div>
 
 				<% if !CONFIG.backends.empty? %>
+                <% if !CONFIG.backend_domains.includes?(env.request.headers["Host"]) %>
 				<div class="h-box">
 					<b>Switch Backend:</b>
 					<% CONFIG.backends.each do | backend | %>
@@ -128,6 +130,7 @@
                     <% end %>
 				</div>
 				<% end %>
+                <% end %>
 
 				<% if CONFIG.banner %>
 					<div class="h-box">
@@ -310,7 +313,10 @@
             </div>
             <hr/>
             <div class="footer-footer">
-			<div class="box">You are currently using Backend: <%= current_backend %></p>
+			<div class="box">You are currently using Backend: <%= current_backend %></div>
+            <% if !current_external_videoplayback_proxy.empty? %>
+            <div class="box">External Videoplayback Proxy: <%= current_external_videoplayback_proxy %></div>
+            <% end %>
                 <span class="left">
                     <% if CONFIG.modified_source_code_url %>
                         <%= translate(locale, "footer_current_version_modified") %>

src/invidious/views/user/preferences.ecr

@@ -126,24 +126,6 @@
                 <input name="save_player_pos" id="save_player_pos" type="checkbox" <% if preferences.save_player_pos %>checked<% end %>>
             </div>
 
-            <% if !CONFIG.ignore_user_tokens %>
-			<div class="pure-control-group">
-                <label for="po_token"><%= translate(locale, "preferences_po_token") %></label>
-                <input name="po_token" id="po_token" type="text" value="<%= preferences.po_token %>">
-            </div>
-
-			<div class="pure-control-group">
-                <label for="visitor_data"><%= translate(locale, "preferences_visitor_data") %></label>
-                <input name="visitor_data" id="visitor_data" type="text" value="<%= preferences.visitor_data %>">
-            </div>
-
-        <% if env.get?("user") %>
-			<div class="pure-control-group">
-				<a href="/generate_tokens?referer=<%= URI.encode_www_form(referer) %>"><%= translate(locale, "Generate po_token and visitor_data for your account") %></a>
-			</div>
-        <% end %>
-        <% end %>
-
             <legend><%= translate(locale, "preferences_category_visual") %></legend>
 
             <div class="pure-control-group">

src/invidious/views/user/subscription_manager.ecr

@@ -37,7 +37,7 @@
             <div class="pure-u-2-5"></div>
             <div class="pure-u-1-5" style="text-align:right">
                 <h3 style="padding-right:0.5em">
-                    <form data-onsubmit="return_false" action="/subscription_ajax?action_remove_subscriptions=1&c=<%= channel.id %>&referer=<%= env.get("current_page") %>" method="post">
+                    <form data-onsubmit="return_false" action="/subscription_ajax?action=remove_subscriptions&c=<%= channel.id %>&referer=<%= env.get("current_page") %>" method="post">
                         <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>">
                         <input style="all:unset" type="submit" data-onclick="remove_subscription" data-ucid="<%= channel.id %>" value="<%= translate(locale, "unsubscribe") %>">
                     </form>

src/invidious/views/user/token_manager.ecr

@@ -29,7 +29,7 @@
         </div>
         <div class="pure-u-1-5" style="text-align:right">
             <h3 style="padding-right:0.5em">
-                <form data-onsubmit="return_false" action="/token_ajax?action_revoke_token=1&session=<%= token[:session] %>&referer=<%= env.get("current_page") %>" method="post">
+                <form data-onsubmit="return_false" action="/token_ajax?action=revoke_token&session=<%= token[:session] %>&referer=<%= env.get("current_page") %>" method="post">
                     <input type="hidden" name="csrf_token" value="<%= HTML.escape(env.get?("csrf_token").try &.as(String) || "") %>">
                     <input style="all:unset" type="submit" data-onclick="revoke_token" data-session="<%= token[:session] %>" value="<%= translate(locale, "revoke") %>">
                 </form>

src/invidious/views/user/tokens.ecr

@@ -1,15 +0,0 @@
-<% content_for "header" do %>
-<title><%= translate(locale, "Invidious token generator") %> - Invidious</title>
-<% end %>
-
-<div class="pure-g">
-    <div class="pure-u-1 pure-u-lg-1-5"></div>
-    <div class="pure-u-1 pure-u-lg-3-5">
-        <div class="h-box">
-		<p>po_token and visitor_data successfully generated!</p>
-		<p>po_token: <%= po_token %></p>
-		<p>visitor_data: <%= visitor_data %></p>
-        </div>
-    </div>
-    <div class="pure-u-1 pure-u-lg-1-5"></div>
-</div>

src/invidious/views/watch.ecr

@@ -1,6 +1,7 @@
 <% ucid = video.ucid %>
 <% title = HTML.escape(video.title) %>
 <% author = HTML.escape(video.author) %>
+<% host = env.request.headers["Host"] %>
 
 
 <% content_for "header" do %>
@@ -8,9 +9,9 @@
 <meta name="description" content="<%= HTML.escape(video.short_description) %>">
 <meta name="keywords" content="<%= video.keywords.join(",") %>">
 <meta property="og:site_name" content="<%= author %> | Invidious">
-<meta property="og:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>">
+<meta property="og:url" content="<%= host %>/watch?v=<%= video.id %>">
 <meta property="og:title" content="<%= title %>">
-<meta property="og:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg">
+<meta property="og:image" content="<%= host %>/vi/<%= video.id %>/maxres.jpg">
 <meta property="og:description" content="<%= HTML.escape(video.short_description) %>">
 <meta property="og:type" content="video.other">
 <!-- This shouldn't be empty, ever. -->
@@ -21,11 +22,11 @@
 <meta property="og:video:width" content="640">
 <meta property="og:video:height" content="360">
 <meta name="twitter:card" content="player">
-<meta name="twitter:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>">
+<meta name="twitter:url" content="<%= host %>/watch?v=<%= video.id %>">
 <meta name="twitter:title" content="<%= title %>">
 <meta name="twitter:description" content="<%= HTML.escape(video.short_description) %>">
-<meta name="twitter:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg">
+<meta name="twitter:image" content="<%= host %>/vi/<%= video.id %>/maxres.jpg">
-<meta name="twitter:player" content="<%= HOST_URL %>/embed/<%= video.id %>">
+<meta name="twitter:player" content="<%= host %>/embed/<%= video.id %>">
 <meta name="twitter:player:width" content="1280">
 <meta name="twitter:player:height" content="720">
 <link rel="alternate" href="https://www.youtube.com/watch?v=<%= video.id %>">
@@ -174,7 +175,7 @@ we're going to need to do it here in order to allow for translations.
             <% if user %>
                 <% playlists = Invidious::Database::Playlists.select_user_created_playlists(user.email) %>
                 <% if !playlists.empty? %>
-                    <form data-onsubmit="return_false" class="pure-form pure-form-stacked" action="/playlist_ajax" method="post" target="_blank">
+                    <form data-onsubmit="return_false" class="pure-form pure-form-stacked" action="/playlist_ajax?action=add_video" method="post" target="_blank">
                         <div class="pure-control-group">
                             <label for="playlist_id"><%= translate(locale, "Add to playlist: ") %></label>
                             <select style="width:100%" name="playlist_id" id="playlist_id">
@@ -185,7 +186,6 @@ we're going to need to do it here in order to allow for translations.
                         </div>
 
                         <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(env.get?("csrf_token").try &.as(String) || "") %>">
-                        <input type="hidden" name="action_add_video" value="1">
                         <input type="hidden" name="video_id" value="<%= video.id %>">
                         <button data-onclick="add_playlist_video" data-id="<%= video.id %>" type="submit" class="pure-button pure-button-primary">
                             <b><%= translate(locale, "Add to playlist") %></b>

src/invidious/yt_backend/connection_pool.cr

@@ -22,12 +22,8 @@ struct YoutubeConnectionPool
       response = yield conn
     rescue ex
       conn.close
+      conn = make_client(url, force_resolve: true)
 
-      conn = HTTP::Client.new(url)
-      conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
-      conn.family = CONFIG.force_resolve
-      conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
-      conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
       response = yield conn
     ensure
       pool.release(conn)
@@ -37,12 +33,15 @@ struct YoutubeConnectionPool
   end
 
   private def build_pool
-    DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
+    options = DB::Pool::Options.new(
-      conn = HTTP::Client.new(url)
+      initial_pool_size: 0,
-      conn.family = CONFIG.force_resolve
+      max_pool_size: capacity,
-      conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
+      max_idle_pool_size: capacity,
-      conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
+      checkout_timeout: timeout
-      conn
+    )
+
+    DB::Pool(HTTP::Client).new(options) do
+      next make_client(url, force_resolve: true)
     end
   end
 end
@@ -62,23 +61,25 @@ def add_yt_headers(request)
   end
 end
 
-def make_client(url : URI, region = nil, force_resolve : Bool = false)
+def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false, use_http_proxy : Bool = true)
   client = HTTP::Client.new(url)
+  client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy && use_http_proxy
 
   # Force the usage of a specific configured IP Family
   if force_resolve
     client.family = CONFIG.force_resolve
+    client.family = Socket::Family::INET if client.family == Socket::Family::UNSPEC
   end
 
-  client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
+  client.before_request { |r| add_yt_headers(r) } if url.host.try &.ends_with?("youtube.com") || force_youtube_headers
   client.read_timeout = 10.seconds
   client.connect_timeout = 10.seconds
 
   return client
 end
 
-def make_client(url : URI, region = nil, force_resolve : Bool = false, &)
+def make_client(url : URI, region = nil, force_resolve : Bool = false, use_http_proxy : Bool = true, &)
-  client = make_client(url, region, force_resolve)
+  client = make_client(url, region, force_resolve: force_resolve, use_http_proxy: use_http_proxy)
   begin
     yield client
   ensure

src/invidious/yt_backend/extractors.cr

@@ -21,6 +21,7 @@ private ITEM_PARSERS = {
   Parsers::ItemSectionRendererParser,
   Parsers::ContinuationItemRendererParser,
   Parsers::HashtagRendererParser,
+  Parsers::LockupViewModelParser,
 }
 
 private alias InitialData = Hash(String, JSON::Any)
@@ -66,6 +67,8 @@ private module Parsers
         author_id = author_fallback.id
       end
 
+      author_thumbnail = item_contents.dig?("channelThumbnailSupportedRenderers", "channelThumbnailWithLinkRenderer", "thumbnail", "thumbnails", 0, "url").try &.as_s
+
       author_verified = has_verified_badge?(item_contents["ownerBadges"]?)
 
       # For live videos (and possibly recently premiered videos) there is no published information.
@@ -147,6 +150,7 @@ private module Parsers
         length_seconds:     length_seconds,
         premiere_timestamp: premiere_timestamp,
         author_verified:    author_verified,
+        author_thumbnail:   author_thumbnail,
         badges:             badges,
       })
     end
@@ -467,9 +471,9 @@ private module Parsers
   # Parses an InnerTube richItemRenderer into a SearchVideo.
   # Returns nil when the given object isn't a RichItemRenderer
   #
-  # A richItemRenderer seems to be a simple wrapper for a videoRenderer, used
+  # A richItemRenderer seems to be a simple wrapper for a various other types,
-  # by the result page for hashtags and for the podcast tab on channels.
+  # used on the hashtags result page and the channel podcast tab. It is located
-  # It is located inside a continuationItems container for hashtags.
+  # itself inside a richGridRenderer container.
   #
   module RichItemRendererParser
     def self.process(item : JSON::Any, author_fallback : AuthorFallback)
@@ -482,6 +486,8 @@ private module Parsers
       child = VideoRendererParser.process(item_contents, author_fallback)
       child ||= ReelItemRendererParser.process(item_contents, author_fallback)
       child ||= PlaylistRendererParser.process(item_contents, author_fallback)
+      child ||= LockupViewModelParser.process(item_contents, author_fallback)
+      child ||= ShortsLockupViewModelParser.process(item_contents, author_fallback)
       return child
     end
 
@@ -496,6 +502,9 @@ private module Parsers
   # reelItemRenderer items are used in the new (2022) channel layout,
   # in the "shorts" tab.
   #
+  # NOTE: As of 10/2024, it might have been fully replaced by shortsLockupViewModel
+  # TODO: Confirm that hypothesis
+  #
   module ReelItemRendererParser
     def self.process(item : JSON::Any, author_fallback : AuthorFallback)
       if item_contents = item["reelItemRenderer"]?
@@ -573,6 +582,137 @@ private module Parsers
         length_seconds:     duration,
         premiere_timestamp: Time.unix(0),
         author_verified:    false,
+        author_thumbnail:   nil,
+        badges:             VideoBadges::None,
+      })
+    end
+
+    def self.parser_name
+      return {{@type.name}}
+    end
+  end
+
+  # Parses an InnerTube lockupViewModel into a SearchPlaylist.
+  # Returns nil when the given object is not a lockupViewModel.
+  #
+  # This structure is present since November 2024 on the "podcasts" and
+  # "playlists" tabs of the channel page. It is usually encapsulated in either
+  # a richItemRenderer or a richGridRenderer.
+  #
+  module LockupViewModelParser
+    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+      if item_contents = item["lockupViewModel"]?
+        return self.parse(item_contents, author_fallback)
+      end
+    end
+
+    private def self.parse(item_contents, author_fallback)
+      playlist_id = item_contents["contentId"].as_s
+
+      thumbnail_view_model = item_contents.dig(
+        "contentImage", "collectionThumbnailViewModel",
+        "primaryThumbnail", "thumbnailViewModel"
+      )
+
+      thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s
+
+      # This complicated sequences tries to extract the following data structure:
+      # "overlays": [{
+      #   "thumbnailOverlayBadgeViewModel": {
+      #     "thumbnailBadges": [{
+      #       "thumbnailBadgeViewModel": {
+      #         "text": "430 episodes",
+      #         "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT"
+      #       }
+      #     }]
+      #   }
+      # }]
+      #
+      # NOTE: this simplistic `.to_i` conversion might not work on larger
+      # playlists and hasn't been tested.
+      video_count = thumbnail_view_model.dig("overlays").as_a
+        .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a)
+        .flatten
+        .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try { |node|
+          {"episodes", "videos"}.any? { |str| node.as_s.ends_with?(str) }
+        })
+        .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false)
+
+      metadata = item_contents.dig("metadata", "lockupMetadataViewModel")
+      title = metadata.dig("title", "content").as_s
+
+      # TODO: Retrieve "updated" info from metadata parts
+      # rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a
+      # parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s)
+      # One of these parts should contain a string like: "Updated 2 days ago"
+
+      # TODO: Maybe add a button to access the first video of the playlist?
+      # item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint")
+      # Available fields: "videoId", "playlistId", "params"
+
+      return SearchPlaylist.new({
+        title:           title,
+        id:              playlist_id,
+        author:          author_fallback.name,
+        ucid:            author_fallback.id,
+        video_count:     video_count || -1,
+        videos:          [] of SearchPlaylistVideo,
+        thumbnail:       thumbnail,
+        author_verified: false,
+      })
+    end
+
+    def self.parser_name
+      return {{@type.name}}
+    end
+  end
+
+  # Parses an InnerTube shortsLockupViewModel into a SearchVideo.
+  # Returns nil when the given object is not a shortsLockupViewModel.
+  #
+  # This structure is present since around October 2024 on the "shorts" tab of
+  # the channel page and likely replaces the reelItemRenderer structure. It is
+  # usually (always?) encapsulated in a richItemRenderer.
+  #
+  module ShortsLockupViewModelParser
+    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+      if item_contents = item["shortsLockupViewModel"]?
+        return self.parse(item_contents, author_fallback)
+      end
+    end
+
+    private def self.parse(item_contents, author_fallback)
+      # TODO: Maybe add support for "oardefault.jpg" thumbnails?
+      # thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s
+      # Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?...
+
+      video_id = item_contents.dig(
+        "onTap", "innertubeCommand", "reelWatchEndpoint", "videoId"
+      ).as_s
+
+      title = item_contents.dig("overlayMetadata", "primaryText", "content").as_s
+
+      view_count = short_text_to_number(
+        item_contents.dig("overlayMetadata", "secondaryText", "content").as_s
+      )
+
+      # Approximate to one minute, as "shorts" generally don't exceed that.
+      # NOTE: The actual duration is not provided by Youtube anymore.
+      # TODO: Maybe use -1 as an error value and handle that on the frontend?
+      duration = 60_i32
+
+      SearchVideo.new({
+        title:              title,
+        id:                 video_id,
+        author:             author_fallback.name,
+        ucid:               author_fallback.id,
+        published:          Time.unix(0),
+        views:              view_count,
+        description_html:   "",
+        length_seconds:     duration,
+        premiere_timestamp: Time.unix(0),
+        author_verified:    false,
+        author_thumbnail:   nil,
         badges:             VideoBadges::None,
       })
     end
@@ -889,7 +1029,7 @@ end
 def extract_items(
   initial_data : InitialData,
   author_fallback : String? = nil,
-  author_id_fallback : String? = nil
+  author_id_fallback : String? = nil,
 ) : {Array(SearchItem), String?}
   items = [] of SearchItem
   continuation = nil

src/invidious/yt_backend/youtube_api.cr

@@ -3,8 +3,6 @@
 #
 
 module YoutubeAPI
-  @@visitor_data : String = ""
-
   extend self
 
   # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history
@@ -204,7 +202,7 @@ module YoutubeAPI
     def initialize(
       *,
       @client_type = ClientType::Web,
-      @region = "US"
+      @region = "US",
     )
     end
 
@@ -322,9 +320,7 @@ module YoutubeAPI
       client_context["client"]["platform"] = platform
     end
 
-    if !@@visitor_data.empty?
+    if CONFIG.visitor_data.is_a?(String)
-      client_context["client"]["visitorData"] = @@visitor_data
-    elsif CONFIG.visitor_data.is_a?(String)
       client_context["client"]["visitorData"] = CONFIG.visitor_data.as(String)
     end
 
@@ -365,7 +361,7 @@ module YoutubeAPI
     browse_id : String,
     *, # Force the following parameters to be passed by name
     params : String,
-    client_config : ClientConfig | Nil = nil
+    client_config : ClientConfig | Nil = nil,
   )
     # JSON Request data, required by the API
     data = {
@@ -460,12 +456,7 @@ module YoutubeAPI
     *, # Force the following parameters to be passed by name
     params : String,
     client_config : ClientConfig | Nil = nil,
-    po_token : String | Nil,
-    visitor_data : String | Nil,
   )
-    if visitor_data
-      @@visitor_data = visitor_data
-    end
     # Playback context, separate because it can be different between clients
     playback_ctx = {
       "html5Preference" => "HTML5_PREF_WANTS",
@@ -491,7 +482,7 @@ module YoutubeAPI
         "contentPlaybackContext" => playback_ctx,
       },
       "serviceIntegrityDimensions" => {
-        "poToken" => po_token || CONFIG.po_token,
+        "poToken" => CONFIG.po_token,
       },
     }
 
@@ -500,8 +491,12 @@ module YoutubeAPI
       data["params"] = params
     end
 
+    if CONFIG.invidious_companion.present?
+      return self._post_invidious_companion("/youtubei/v1/player", data)
+    else
       return self._post_json("/youtubei/v1/player", data, client_config)
     end
+  end
 
   ####################################################################
   # resolve_url(url, client_config?)
@@ -557,7 +552,7 @@ module YoutubeAPI
   def search(
     search_query : String,
     params : String,
-    client_config : ClientConfig | Nil = nil
+    client_config : ClientConfig | Nil = nil,
   )
     # JSON Request data, required by the API
     data = {
@@ -583,7 +578,7 @@ module YoutubeAPI
 
   def get_transcript(
     params : String,
-    client_config : ClientConfig | Nil = nil
+    client_config : ClientConfig | Nil = nil,
   ) : Hash(String, JSON::Any)
     data = {
       "context" => self.make_context(client_config),
@@ -625,9 +620,7 @@ module YoutubeAPI
       headers["User-Agent"] = user_agent
     end
 
-    if !@@visitor_data.empty?
+    if CONFIG.visitor_data.is_a?(String)
-      headers["X-Goog-Visitor-Id"] = @@visitor_data
-    elsif CONFIG.visitor_data.is_a?(String)
       headers["X-Goog-Visitor-Id"] = CONFIG.visitor_data.as(String)
     end
 
@@ -639,6 +632,11 @@ module YoutubeAPI
     # Send the POST request
     body = YT_POOL.client() do |client|
       client.post(url, headers: headers, body: data.to_json) do |response|
+        if response.status_code != 200
+          raise InfoException.new("Error: non 200 status code. Youtube API returned \
+            status code #{response.status_code}. See <a href=\"https://docs.invidious.io/youtube-errors-explained/\"> \
+            https://docs.invidious.io/youtube-errors-explained/</a> for troubleshooting.")
+        end
         self._decompress(response.body_io, response.headers["Content-Encoding"]?)
       end
     end
@@ -663,6 +661,51 @@ module YoutubeAPI
     return initial_data
   end
 
+  ####################################################################
+  # _post_invidious_companion(endpoint, data)
+  #
+  # Internal function that does the actual request to Invidious companion
+  # and handles errors.
+  #
+  # The requested data is an endpoint (URL without the domain part)
+  # and the data as a Hash object.
+  #
+  def _post_invidious_companion(
+    endpoint : String,
+    data : Hash
+  ) : Hash(String, JSON::Any)
+    headers = HTTP::Headers{
+      "Content-Type"  => "application/json; charset=UTF-8",
+      "Authorization" => "Bearer #{CONFIG.invidious_companion_key}",
+    }
+
+    # Logging
+    LOGGER.debug("Invidious companion: Using endpoint: \"#{endpoint}\"")
+    LOGGER.trace("Invidious companion: POST data: #{data}")
+
+    # Send the POST request
+
+    begin
+      invidious_companion = CONFIG.invidious_companion.sample
+      response = make_client(invidious_companion.private_url, use_http_proxy: false,
+        &.post(endpoint, headers: headers, body: data.to_json))
+      body = response.body
+      if (response.status_code != 200)
+        raise Exception.new(
+          "Error while communicating with Invidious companion: \
+          status code: #{response.status_code} and body: #{body.dump}"
+        )
+      end
+    rescue ex
+      raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found"))
+    end
+
+    # Convert result to Hash
+    initial_data = JSON.parse(body).as_h
+
+    return initial_data
+  end
+
   ####################################################################
   # _decompress(body_io, headers)
   #
@@ -681,7 +724,7 @@ module YoutubeAPI
       # Multiple encodings can be combined, and are listed in the order
       # in which they were applied. E.g: "deflate, gzip" means that the
       # content must be first "gunzipped", then "defated".
-      encodings.split(',').reverse.each do |enc|
+      encodings.split(',').reverse!.each do |enc|
         case enc.strip(' ')
         when "gzip"
           body_io = Compress::Gzip::Reader.new(body_io, sync_close: true)