Update CHANGELOG.md

Signed-off-by: Fijxu <fijxu@nadeko.net>

.forgejo/workflows/ci.yml

@@ -8,6 +8,8 @@ on:
   push:
     branches:
       - "master"
+      - "experimental"
+      - "experimental2"
 
 jobs:
   build:

.github/workflows/ci.yml

@@ -38,10 +38,11 @@ jobs:
       matrix:
         stable: [true]
         crystal:
-          - 1.9.2
           - 1.10.1
           - 1.11.2
           - 1.12.1
+          - 1.13.2
+          - 1.14.0
         include:
           - crystal: nightly
             stable: false
@@ -51,6 +52,11 @@ jobs:
         with:
           submodules: true
 
+      - name: Install required APT packages
+        run: |
+          sudo apt install -y libsqlite3-dev 
+        shell: bash
+
       - name: Install Crystal
         uses: crystal-lang/install-crystal@v1.8.0
         with:

CHANGELOG.md

@@ -5,6 +5,12 @@
 
 ### Full list of pull requests merged since the last release (newest first)
 
+* Add "Filipino (auto-generated)" to the list of caption languages ([#4995], by @SamantazFox)
+* Makefile: Add MT option to enable the 'preview_mt' flag ([#4993], by @SamantazFox)
+* SigHelper: Reconnect to signature helper ([#4991], thanks @Fijxu)
+* Fix player menus hiding onHover ready ([#4750], thanks @giacomocerquone)
+* Use connection pools when requesting images from YouTube ([#4326], thanks @syeopite)
+* Add support for using Invidious through a HTTP Proxy ([#4270], thanks @syeopite)
 * Search: Fix 'youtu.be' URLs in sanitizer ([#4894], by @SamantazFox)
 * Ameba: Disable Style/RedundantNext rule ([#4888], thanks @syeopite)
 * Playlists: Fix 'invalid byte sequence' error when subscribing ([#4887], thanks @DmitrySandalov)
@@ -22,7 +28,10 @@
 
 [#4122]: https://github.com/iv-org/invidious/pull/4122
 [#4193]: https://github.com/iv-org/invidious/pull/4193
+[#4270]: https://github.com/iv-org/invidious/pull/4270
+[#4326]: https://github.com/iv-org/invidious/pull/4326
 [#4652]: https://github.com/iv-org/invidious/pull/4652
+[#4750]: https://github.com/iv-org/invidious/pull/4750
 [#4850]: https://github.com/iv-org/invidious/pull/4850
 [#4862]: https://github.com/iv-org/invidious/pull/4862
 [#4863]: https://github.com/iv-org/invidious/pull/4863
@@ -33,6 +42,9 @@
 [#4928]: https://github.com/iv-org/invidious/pull/4928
 [#4930]: https://github.com/iv-org/invidious/pull/4930
 [#4942]: https://github.com/iv-org/invidious/pull/4942
+[#4991]: https://github.com/iv-org/invidious/pull/4991
+[#4993]: https://github.com/iv-org/invidious/pull/4993
+[#4995]: https://github.com/iv-org/invidious/pull/4995
 
 
 ## v2.20240825.2 (2024-08-26)

Makefile

@@ -7,6 +7,11 @@ STATIC   := 0
 
 NO_DBG_SYMBOLS := 0
 
+# Enable multi-threading.
+# Warning: Experimental feature!!
+# invidious is not stable when MT is enabled.
+MT := 0
+
 
 FLAGS ?=
 
@@ -19,6 +24,10 @@ ifeq ($(STATIC), 1)
   FLAGS += --static
 endif
 
+ifeq ($(MT), 1)
+  FLAGS += -Dpreview_mt
+endif
+
 
 ifeq ($(NO_DBG_SYMBOLS), 1)
   FLAGS += --no-debug

assets/css/player.css

@@ -68,6 +68,7 @@
 
 .video-js.player-style-youtube .vjs-menu-button-popup .vjs-menu {
   margin-bottom: 2em;
+  padding-top: 2em
 }
 
 .video-js.player-style-youtube .vjs-progress-control .vjs-progress-holder, .video-js.player-style-youtube .vjs-progress-control {height: 5px;

config/config.example.yml

@@ -173,6 +173,17 @@ https_only: false
 ##
 #force_resolve:
 
+##
+## Configuration for using a HTTP proxy
+##
+## If unset, then no HTTP proxy will be used.
+## 
+http_proxy:
+  user:
+  password:
+  host:
+  port:
+
 
 ##
 ## Use Innertube's transcripts API instead of timedtext for closed captions

crystal_formatters.py

@@ -0,0 +1,62 @@
+import lldb
+
+class CrystalArraySyntheticProvider:
+    def __init__(self, valobj, internal_dict):
+        self.valobj = valobj
+        self.buffer = None
+        self.size = 0
+
+    def update(self):
+        if self.valobj.type.is_pointer:
+            self.valobj = self.valobj.Dereference()
+        self.size = int(self.valobj.child[0].value)
+        self.type = self.valobj.type
+        self.buffer = self.valobj.child[3]
+
+    def num_children(self):
+        size = 0 if self.size is None else self.size
+        return size
+
+    def get_child_index(self, name):
+        try:
+            return int(name.lstrip('[').rstrip(']'))
+        except:
+            return -1
+
+    def get_child_at_index(self,index):
+        if index >= self.size:
+            return None
+        try:
+            elementType = self.buffer.type.GetPointeeType()
+            offset = elementType.size * index
+            return self.buffer.CreateChildAtOffset('[' + str(index) + ']', offset, elementType)
+        except Exception as e:
+            print('Got exception %s' % (str(e)))
+            return None
+
+def findType(name, module):
+    cachedTypes = module.GetTypes()
+    for idx in range(cachedTypes.GetSize()):
+        type = cachedTypes.GetTypeAtIndex(idx)
+        if type.name == name:
+            return type
+    return None
+
+
+def CrystalString_SummaryProvider(value, dict):
+    error = lldb.SBError()
+    if value.TypeIsPointerType():
+        value = value.Dereference()
+    process = value.GetTarget().GetProcess()
+    byteSize = int(value.child[0].value)
+    len = int(value.child[1].value)
+    len = byteSize or len
+    strAddr = value.child[2].load_addr
+    val = process.ReadCStringFromMemory(strAddr, len + 1, error)
+    return '"%s"' % val
+
+
+def __lldb_init_module(debugger, dict):
+    debugger.HandleCommand(r'type synthetic add -l crystal_formatters.CrystalArraySyntheticProvider -x "^Array\(.+\)(\s*\**)?" -w Crystal')
+    debugger.HandleCommand(r'type summary add -F crystal_formatters.CrystalString_SummaryProvider -x "^(String|\(String \| Nil\))(\s*\**)?$" -w Crystal')
+    debugger.HandleCommand(r'type category enable Crystal')

locales/en-US.json

@@ -287,6 +287,7 @@
     "Esperanto": "Esperanto",
     "Estonian": "Estonian",
     "Filipino": "Filipino",
+    "Filipino (auto-generated)": "Filipino (auto-generated)",
     "Finnish": "Finnish",
     "French": "French",
     "French (auto-generated)": "French (auto-generated)",

mocks

@@ -1 +1 @@
-Subproject commit 11ec372f72747c09d48ffef04843f72be67d5b54
+Subproject commit b55d58dea94f7144ff0205857dfa70ec14eaa872

shard.lock

@@ -10,7 +10,7 @@ shards:
 
   backtracer:
     git: https://github.com/sija/backtracer.cr.git
-    version: 1.2.1
+    version: 1.2.2
 
   db:
     git: https://github.com/crystal-lang/crystal-db.git
@@ -20,6 +20,14 @@ shards:
     git: https://github.com/crystal-loot/exception_page.git
     version: 0.2.2
 
+  inotify:
+    git: https://github.com/petoem/inotify.cr.git
+    version: 1.0.3
+
+  http_proxy:
+    git: https://github.com/mamantoha/http_proxy.git
+    version: 0.10.3
+
   kemal:
     git: https://github.com/kemalcr/kemal.git
     version: 1.1.2
@@ -50,7 +58,7 @@ shards:
 
   spectator:
     git: https://github.com/icy-arctic-fox/spectator.git
-    version: 0.10.4
+    version: 0.10.6
 
   sqlite3:
     git: https://github.com/crystal-lang/crystal-sqlite3.git

shard.yml

@@ -30,6 +30,12 @@ dependencies:
     version: ~> 0.1.1
   redis:
     github: stefanwille/crystal-redis
+  inotify:
+    github: petoem/inotify.cr
+    version: 1.0.3
+  http_proxy:
+    github: mamantoha/http_proxy
+    version: ~> 0.10.3
 
 development_dependencies:
   spectator:

spec/invidious/videos/regular_videos_extract_spec.cr

@@ -17,8 +17,8 @@ Spectator.describe "parse_video_info" do
     # Basic video infos
 
     expect(info["title"].as_s).to eq("I Gave My 100,000,000th Subscriber An Island")
-    expect(info["views"].as_i).to eq(126_573_823)
-    expect(info["likes"].as_i).to eq(5_157_654)
+    expect(info["views"].as_i).to eq(220_226_287)
+    expect(info["likes"].as_i).to eq(6_870_691)
 
     # For some reason the video length from VideoDetails and the
     # one from microformat differs by 1s...
@@ -48,12 +48,12 @@ Spectator.describe "parse_video_info" do
 
     expect(info["relatedVideos"].as_a.size).to eq(20)
 
-    expect(info["relatedVideos"][0]["id"]).to eq("Hwybp38GnZw")
-    expect(info["relatedVideos"][0]["title"]).to eq("I Built Willy Wonka's Chocolate Factory!")
+    expect(info["relatedVideos"][0]["id"]).to eq("krsBRQbOPQ4")
+    expect(info["relatedVideos"][0]["title"]).to eq("$1 vs $250,000,000 Private Island!")
     expect(info["relatedVideos"][0]["author"]).to eq("MrBeast")
     expect(info["relatedVideos"][0]["ucid"]).to eq("UCX6OQ3DkcsbYNE6H8uQQuVA")
-    expect(info["relatedVideos"][0]["view_count"]).to eq("179877630")
-    expect(info["relatedVideos"][0]["short_view_count"]).to eq("179M")
+    expect(info["relatedVideos"][0]["view_count"]).to eq("230617484")
+    expect(info["relatedVideos"][0]["short_view_count"]).to eq("230M")
     expect(info["relatedVideos"][0]["author_verified"]).to eq("true")
 
     # Description
@@ -76,11 +76,11 @@ Spectator.describe "parse_video_info" do
     expect(info["ucid"].as_s).to eq("UCX6OQ3DkcsbYNE6H8uQQuVA")
 
     expect(info["authorThumbnail"].as_s).to eq(
-      "https://yt3.ggpht.com/ytc/AL5GRJVuqw82ERvHzsmBxL7avr1dpBtsVIXcEzBPZaloFg=s48-c-k-c0x00ffffff-no-rj"
+      "https://yt3.ggpht.com/fxGKYucJAVme-Yz4fsdCroCFCrANWqw0ql4GYuvx8Uq4l_euNJHgE-w9MTkLQA805vWCi-kE0g=s48-c-k-c0x00ffffff-no-rj"
     )
 
     expect(info["authorVerified"].as_bool).to be_true
-    expect(info["subCountText"].as_s).to eq("143M")
+    expect(info["subCountText"].as_s).to eq("320M")
   end
 
   it "parses a regular video with no descrition/comments" do
@@ -99,8 +99,8 @@ Spectator.describe "parse_video_info" do
     # Basic video infos
 
     expect(info["title"].as_s).to eq("Chris Rea - Auberge")
-    expect(info["views"].as_i).to eq(10_943_126)
-    expect(info["likes"].as_i).to eq(0)
+    expect(info["views"].as_i).to eq(14_324_584)
+    expect(info["likes"].as_i).to eq(35_870)
     expect(info["lengthSeconds"].as_i).to eq(283_i64)
     expect(info["published"].as_s).to eq("2012-05-21T00:00:00Z")
 
@@ -132,14 +132,14 @@ Spectator.describe "parse_video_info" do
 
     # Related videos
 
-    expect(info["relatedVideos"].as_a.size).to eq(19)
+    expect(info["relatedVideos"].as_a.size).to eq(20)
 
-    expect(info["relatedVideos"][0]["id"]).to eq("Ww3KeZ2_Yv4")
-    expect(info["relatedVideos"][0]["title"]).to eq("Chris Rea")
-    expect(info["relatedVideos"][0]["author"]).to eq("PanMusic")
-    expect(info["relatedVideos"][0]["ucid"]).to eq("UCsKAPSuh1iNbLWUga_igPyA")
-    expect(info["relatedVideos"][0]["view_count"]).to eq("31581")
-    expect(info["relatedVideos"][0]["short_view_count"]).to eq("31K")
+    expect(info["relatedVideos"][0]["id"]).to eq("gUUdQfnshJ4")
+    expect(info["relatedVideos"][0]["title"]).to eq("Chris Rea - The Road To Hell 1989 Full Version")
+    expect(info["relatedVideos"][0]["author"]).to eq("NEA ZIXNH")
+    expect(info["relatedVideos"][0]["ucid"]).to eq("UCYMEOGcvav3gCgImK2J07CQ")
+    expect(info["relatedVideos"][0]["view_count"]).to eq("53298661")
+    expect(info["relatedVideos"][0]["short_view_count"]).to eq("53M")
     expect(info["relatedVideos"][0]["author_verified"]).to eq("false")
 
     # Description
@@ -156,11 +156,13 @@ Spectator.describe "parse_video_info" do
 
     # Author infos
 
-    expect(info["author"].as_s).to eq("ChrisReaOfficial")
+    expect(info["author"].as_s).to eq("ChrisReaVideos")
     expect(info["ucid"].as_s).to eq("UC_5q6nWPbD30-y6oiWF_oNA")
 
-    expect(info["authorThumbnail"].as_s).to be_empty
+    expect(info["authorThumbnail"].as_s).to eq(
+      "https://yt3.ggpht.com/ytc/AIdro_n71nsegpKfjeRKwn1JJmK5IVMh_7j5m_h3_1KnUUg=s48-c-k-c0x00ffffff-no-rj"
+    )
     expect(info["authorVerified"].as_bool).to be_false
-    expect(info["subCountText"].as_s).to eq("-")
+    expect(info["subCountText"].as_s).to eq("3.11K")
   end
 end

src/invidious.cr

@@ -23,6 +23,7 @@ require "kilt"
 require "./ext/kemal_content_for.cr"
 require "./ext/kemal_static_file_handler.cr"
 
+require "http_proxy"
 require "athena-negotiation"
 require "openssl/hmac"
 require "option_parser"
@@ -32,6 +33,7 @@ require "yaml"
 require "compress/zip"
 require "protodec/utils"
 require "redis"
+require "inotify"
 
 require "./invidious/database/*"
 require "./invidious/database/migrations/*"
@@ -59,6 +61,19 @@ end
 alias IV = Invidious
 
 CONFIG = Config.load
+
+Signal::HUP.trap do
+  Config.reload
+end
+
+{% if flag?(:linux) %}
+  if CONFIG.reload_config_automatically
+    Inotify.watch("config/config.yml") do |event|
+      Config.reload
+    end
+  end
+{% end %}
+
 HMAC_KEY = CONFIG.hmac_key
 
 PG_DB    = DB.open CONFIG.database_url
@@ -71,7 +86,9 @@ ARCHIVE_URL = URI.parse("https://archive.org")
 PUBSUB_URL      = URI.parse("https://pubsubhubbub.appspot.com")
 REDDIT_URL      = URI.parse("https://www.reddit.com")
 YT_URL          = URI.parse("https://www.youtube.com")
+PUBSUB_HOST_URL = CONFIG.pubsub_domain
 HOST_URL        = make_host_url(Kemal.config)
+EXT_VIDEOP_LIST = gen_videoplayback_proxy_list()
 
 CHARS_SAFE         = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
 TEST_IDS           = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
@@ -98,6 +115,10 @@ SOFTWARE = {
 
 YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
 
+# Image request pool
+
+GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size)
+
 # CLI
 Kemal.config.extra_options do |parser|
   parser.banner = "Usage: invidious [arguments]"
@@ -188,10 +209,14 @@ Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new
 
 Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new
 
-if CONFIG.external_videoplayback_proxy
+if !CONFIG.external_videoplayback_proxy.empty?
   Invidious::Jobs.register Invidious::Jobs::CheckExternalProxy.new
 end
 
+if CONFIG.refresh_tokens
+  Invidious::Jobs.register Invidious::Jobs::RefreshTokens.new
+end
+
 Invidious::Jobs.start_all
 
 def popular_videos

src/invidious/channels/channels.cr

@@ -251,8 +251,8 @@ def fetch_channel(ucid, pull_all_videos : Bool)
       LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
       if CONFIG.enable_user_notifications
         Invidious::Database::Users.add_notification(video)
-      else
-        Invidious::Database::Users.feed_needs_update(video)
+        #   else
+        #     Invidious::Database::Users.feed_needs_update(video)
       end
     else
       LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
@@ -287,8 +287,8 @@ def fetch_channel(ucid, pull_all_videos : Bool)
           if was_insert
             if CONFIG.enable_user_notifications
               Invidious::Database::Users.add_notification(video)
-            else
-              Invidious::Database::Users.feed_needs_update(video)
+              # else
+              #   Invidious::Database::Users.feed_needs_update(video)
             end
           end
         end

src/invidious/config.cr

@@ -45,6 +45,8 @@ struct ConfigPreferences
   property vr_mode : Bool = true
   property show_nick : Bool = true
   property save_player_pos : Bool = false
+  property po_token : String = ""
+  property visitor_data : String = ""
 
   def to_tuple
     {% begin %}
@@ -55,6 +57,15 @@ struct ConfigPreferences
   end
 end
 
+struct HTTPProxyConfig
+  include YAML::Serializable
+
+  property user : String
+  property password : String
+  property host : String
+  property port : Int32
+end
+
 class Config
   include YAML::Serializable
 
@@ -87,10 +98,14 @@ class Config
 
   # Used to tell Invidious it is behind a proxy, so links to resources should be https://
   property https_only : Bool?
+  # Enable or disable CSP
+  property csp : Bool? = true
   # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
   property hmac_key : String = ""
   # Domain to be used for links to resources on the site where an absolute URL is required
   property domain : String?
+  # Materialious redirects
+  property materialious_domain : String?
   # Alternative domains. You can add other domains, like TOR and I2P addresses
   property alternative_domains : Array(String) = [] of String
   property donation_url : String?
@@ -151,6 +166,8 @@ class Config
   property host_binding : String = "0.0.0.0"
   # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
   property pool_size : Int32 = 100
+  # HTTP Proxy configuration
+  property http_proxy : HTTPProxyConfig? = nil
 
   # Use Innertube's transcripts API instead of timedtext for closed captions
   property use_innertube_for_captions : Bool = false
@@ -176,10 +193,20 @@ class Config
   # of the backend
   property backends_delimiter : String = "|"
 
-  property external_videoplayback_proxy : String?
+  # External videoplayback proxies list. They should include `https://`
+  # at the start of the URI
+  property external_videoplayback_proxy : Array(NamedTuple(url: String, balance: Bool)) = [] of NamedTuple(url: String, balance: Bool)
 
-  # Materialious redirects
-  property materialious_domain : String?
+  # Job to refresh tokens from a Redis compatible DB
+  property refresh_tokens : Bool = true
+
+  property pubsub_domain : String = ""
+
+  property ignore_user_tokens : Bool = false
+
+  {% if flag?(:linux) %}
+    property reload_config_automatically : Bool = true
+  {% end %}
 
   def disabled?(option)
     case disabled = CONFIG.disable_proxy
@@ -196,6 +223,64 @@ class Config
     end
   end
 
+  def self.reload
+    LOGGER.info("Config: Reloading configuration")
+    # Load config from file or YAML string env var
+    env_config_file = "INVIDIOUS_CONFIG_FILE"
+    env_config_yaml = "INVIDIOUS_CONFIG"
+
+    config_file = ENV.has_key?(env_config_file) ? ENV.fetch(env_config_file) : "config/config.yml"
+    config_yaml = ENV.has_key?(env_config_yaml) ? ENV.fetch(env_config_yaml) : File.read(config_file)
+
+    begin
+      config = Config.from_yaml(config_yaml)
+    rescue ex
+      LOGGER.error("Config: Error when reloading configuration: '#{ex.message}'")
+      config = CONFIG
+    end
+
+    # TODO: Preserve old config and don't exit on fail
+    {% for ivar in Config.instance_vars %}
+	  CONFIG.{{ivar}} = config.{{ivar}}
+	  {% env_id = "INVIDIOUS_#{ivar.id.upcase}" %}
+
+        if ENV.has_key?({{env_id}})
+            env_value = ENV.fetch({{env_id}})
+            success = false
+
+            # Use YAML converter if specified
+            {% ann = ivar.annotation(::YAML::Field) %}
+            {% if ann && ann[:converter] %}
+                CONFIG.{{ivar.id}} = {{ann[:converter]}}.from_yaml(YAML::ParseContext.new, YAML::Nodes.parse(ENV.fetch({{env_id}})).nodes[0])
+                success = true
+
+            # Use regular YAML parser otherwise
+            {% else %}
+                {% ivar_types = ivar.type.union? ? ivar.type.union_types : [ivar.type] %}
+                # Sort types to avoid parsing nulls and numbers as strings
+                {% ivar_types = ivar_types.sort_by { |ivar_type| ivar_type == Nil ? 0 : ivar_type == Int32 ? 1 : 2 } %}
+                {{ivar_types}}.each do |ivar_type|
+                    if !success
+                        begin
+                            CONFIG.{{ivar.id}} = ivar_type.from_yaml(env_value)
+                            success = true
+                        rescue
+                            # nop
+                        end
+                    end
+                end
+            {% end %}
+
+            # Exit on fail
+            if !success
+				LOGGER.error("Config: Error when reloading environment variables for the configuration, exiting (fixme!)")
+                exit(1)
+            end
+        end
+	{% end %}
+    LOGGER.info("Config: Reload successfull")
+  end
+
   def self.load
     # Load config from file or YAML string env var
     env_config_file = "INVIDIOUS_CONFIG_FILE"

src/invidious/database/users.cr

@@ -154,15 +154,17 @@ module Invidious::Database::Users
   #  Update (misc)
   # -------------------
 
-  def feed_needs_update(video : ChannelVideo)
-    request = <<-SQL
-      UPDATE users
-      SET feed_needs_update = true
-      WHERE $1 = ANY(subscriptions)
-    SQL
+  # Feeds never need update. PubSubHubBub is the one that sends videos to
+  # invidious.
+  #   def feed_needs_update(video : ChannelVideo)
+  #     request = <<-SQL
+  #       UPDATE users
+  #       SET feed_needs_update = true
+  #       WHERE $1 = ANY(subscriptions)
+  #     SQL
 
-    PG_DB.exec(request, video.ucid)
-  end
+  #     PG_DB.exec(request, video.ucid)
+  #   end
 
   def update_preferences(user : User)
     request = <<-SQL

src/invidious/database/videos.cr

@@ -10,8 +10,8 @@ module Invidious::Database::Videos
       ON CONFLICT (id) DO NOTHING
     SQL
 
-    REDIS_DB.set(video.id, video.info.to_json, ex: 3600)
-    REDIS_DB.set(video.id + ":time", video.updated, ex: 3600)
+    REDIS_DB.set(video.id, video.info.to_json, ex: 14400)
+    REDIS_DB.set(video.id + ":time", video.updated, ex: 14400)
   end
 
   def delete(id)

src/invidious/helpers/crystal_class_overrides.cr

@@ -18,6 +18,40 @@ end
 class HTTP::Client
   property family : Socket::Family = Socket::Family::UNSPEC
 
+  # Override stdlib to automatically initialize proxy if configured
+  #
+  # Accurate as of crystal 1.12.1
+
+  def initialize(@host : String, port = nil, tls : TLSContext = nil)
+    check_host_only(@host)
+
+    {% if flag?(:without_openssl) %}
+      if tls
+        raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time"
+      end
+      @tls = nil
+    {% else %}
+      @tls = case tls
+             when true
+               OpenSSL::SSL::Context::Client.new
+             when OpenSSL::SSL::Context::Client
+               tls
+             when false, nil
+               nil
+             end
+    {% end %}
+
+    @port = (port || (@tls ? 443 : 80)).to_i
+
+    self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+  end
+
+  def initialize(@io : IO, @host = "", @port = 80)
+    @reconnect = false
+
+    self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+  end
+
   private def io
     io = @io
     return io if io

src/invidious/helpers/redis_tokens.cr

@@ -0,0 +1,49 @@
+module Tokens
+  extend self
+  @@po_token : String | Nil
+  @@visitor_data : String | Nil
+
+  def refresh_tokens
+    @@po_token = REDIS_DB.get("invidious:po_token")
+    @@visitor_data = REDIS_DB.get("invidious:visitor_data")
+    LOGGER.debug("RefreshTokens: Tokens are:")
+    LOGGER.debug("RefreshTokens: po_token: #{@@po_token}")
+    LOGGER.debug("RefreshTokens: visitor_data: #{@@visitor_data}")
+  end
+
+  def get_tokens
+    return {@@po_token, @@visitor_data}
+  end
+
+  def get_po_token
+    return @@po_token
+  end
+
+  def get_visitor_data
+    return @@visitor_data
+  end
+
+  def generate_tokens(user : String)
+    po_token = ""
+    visitor_data = ""
+    attempts = 0
+
+    LOGGER.debug("Generating po_token and visitor_data for user: '#{user}'")
+    REDIS_DB.publish("generate-token", "#{user}")
+
+    while REDIS_DB.get("invidious:#{user}:po_token").nil? && REDIS_DB.get("invidious:#{user}:visitor_data").nil?
+      if attempts > 50
+        break
+      end
+      LOGGER.debug("Waiting for tokens to arrive at redis for user: '#{user}'")
+      attempts += 1
+      sleep 250.milliseconds
+    end
+
+    po_token = REDIS_DB.get("invidious:#{user}:po_token")
+    visitor_data = REDIS_DB.get("invidious:#{user}:visitor_data")
+
+    LOGGER.debug("Tokens successfully generated for user: '#{user}'")
+    return {po_token, visitor_data}
+  end
+end

src/invidious/helpers/sig_helper.cr

@@ -175,7 +175,6 @@ module Invidious::SigHelper
     @queue = {} of TransactionID => Transaction
 
     @conn : Connection
-
     @uri_or_path : String
 
     def initialize(@uri_or_path)
@@ -201,7 +200,7 @@ module Invidious::SigHelper
                 @conn = Connection.new(@uri_or_path)
                 LOGGER.info("SigHelper: Reconnected to SigHelper!")
               rescue ex
-                LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}' retrying")
+                LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying")
                 sleep 500.milliseconds
                 next
               end

src/invidious/helpers/utils.cr

@@ -294,7 +294,7 @@ def subscribe_pubsub(topic, key)
   signature = "#{time}:#{nonce}"
 
   body = {
-    "hub.callback"      => "#{HOST_URL}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}",
+    "hub.callback"      => "#{PUBSUB_HOST_URL}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}",
     "hub.topic"         => "https://www.youtube.com/xml/feeds/videos.xml?#{topic}",
     "hub.verify"        => "async",
     "hub.mode"          => "subscribe",
@@ -383,3 +383,17 @@ def parse_link_endpoint(endpoint : JSON::Any, text : String, video_id : String)
   end
   return text
 end
+
+# Generates a list of external videoplayback proxies for 
+# CSP
+def gen_videoplayback_proxy_list
+  if !CONFIG.external_videoplayback_proxy.empty?
+    external_videoplayback_proxy = ""
+    CONFIG.external_videoplayback_proxy.each do |proxy|
+      external_videoplayback_proxy += " #{proxy[:url]}"
+    end
+  else
+    external_videoplayback_proxy = ""
+  end
+  return external_videoplayback_proxy
+end

src/invidious/http_server/utils.cr

@@ -4,6 +4,53 @@ module Invidious::HttpServer
   module Utils
     extend self
 
+    @@proxy_list : Array(String) = [] of String
+    @@current_proxy : String = ""
+    @@count : Int64 = Time.utc.to_unix
+
+    def check_external_proxy
+      CONFIG.external_videoplayback_proxy.each do |proxy|
+        begin
+          response = HTTP::Client.get("#{proxy[:url]}/health")
+          if response.status_code == 200
+            if @@proxy_list.includes?(proxy[:url])
+              next
+            end
+            if proxy[:balance]
+              @@proxy_list << proxy[:url]
+              LOGGER.debug("CheckExternalProxy: Adding proxy '#{proxy[:url]}' to the list of proxies")
+            end
+            break if proxy[:balance] == false && !@@proxy_list.empty?
+            @@proxy_list << proxy[:url]
+          end
+        rescue
+          if @@proxy_list.includes?(proxy[:url])
+            LOGGER.debug("CheckExternalProxy: Proxy '#{proxy[:url]}' is not available, removing it from the list of proxies")
+            @@proxy_list.delete(proxy[:url])
+          end
+          LOGGER.debug("CheckExternalProxy: Proxy '#{proxy[:url]}' is not available")
+        end
+      end
+      LOGGER.trace("CheckExternalProxy: List of proxies:")
+      LOGGER.trace("#{@@proxy_list.inspect}")
+    end
+
+    # TODO: If the function is called many times, it will return a random
+    # proxy from the list. That is not how it should be.
+    # It should return the same proxy, in multiple function calls
+    def select_proxy
+      if (@@count - (Time.utc.to_unix - 30)) <= 0
+        return if @@proxy_list.size <= 0
+        @@current_proxy = @@proxy_list[Random.rand(@@proxy_list.size)]
+        LOGGER.debug("Current proxy is: '#{@@current_proxy}'")
+        @@count = Time.utc.to_unix
+      end
+    end
+
+    def get_external_proxy
+      return @@current_proxy
+    end
+
     def proxy_video_url(raw_url : String, *, region : String? = nil, absolute : Bool = false)
       url = URI.parse(raw_url)
 
@@ -14,7 +61,11 @@ module Invidious::HttpServer
       url.query_params = params
 
       if absolute
+        if !(proxy = get_external_proxy()).empty?
+          return "#{proxy}#{url.request_target}"
+        else
           return "#{HOST_URL}#{url.request_target}"
+        end
       else
         return url.request_target
       end

src/invidious/jobs/check_external_proxy.cr

@@ -4,9 +4,10 @@ class Invidious::Jobs::CheckExternalProxy < Invidious::Jobs::BaseJob
 
   def begin
     loop do
-      Invidious::Routes::API::Manifest.check_external_proxy
-      LOGGER.info("CheckExternalProxy: Done, sleeping for 1 minute")
-      sleep 1.minutes
+      HttpServer::Utils.check_external_proxy
+      HttpServer::Utils.select_proxy
+      LOGGER.info("CheckExternalProxy: Done, sleeping for 15 seconds")
+      sleep 15.seconds
       Fiber.yield
     end
   end

src/invidious/jobs/refresh_tokens.cr

@@ -0,0 +1,13 @@
+class Invidious::Jobs::RefreshTokens < Invidious::Jobs::BaseJob
+  def initialize
+  end
+
+  def begin
+    loop do
+      Tokens.refresh_tokens
+      LOGGER.info("RefreshTokens: Done, sleeping for 5 seconds")
+      sleep 5.seconds
+      Fiber.yield
+    end
+  end
+end

src/invidious/jobs/subscribe_to_feeds_job.cr

@@ -30,6 +30,8 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
           spawn do
             begin
               response = subscribe_pubsub(ucid, hmac_key)
+              LOGGER.debug("SubscribeToFeedsJob: Subscribed to #{ucid}.")
+              LOGGER.trace("SubscribeToFeedsJob: response.body: #{response.body}")
 
               if response.status_code >= 400
                 LOGGER.error("SubscribeToFeedsJob: #{ucid} : #{response.body}")

src/invidious/routes/account.cr

@@ -349,4 +349,40 @@ module Invidious::Routes::Account
       return "{}"
     end
   end
+
+  # -------------------
+  #  poToken and visitorData tokens generation
+  # -------------------
+
+  # Generates a poToken & visitorData for the user, server side
+  def generate_tokens(env)
+    locale = env.get("preferences").as(Preferences).locale
+    preferences = env.get("preferences").as(Preferences)
+
+    user = env.get? "user"
+    sid = env.get? "sid"
+    referer = get_referer(env)
+
+    if !user
+      return env.redirect referer
+    end
+
+    user = user.as(User)
+
+    po_token, visitor_data = Tokens.generate_tokens(user.email)
+
+    if po_token.nil? || visitor_data.nil?
+      return error_template(500, "Internal server error. Please submit an issue here IF THE ISSUE PERSISTS: https://git.nadeko.net/Fijxu/invidious/issues")
+    end
+
+    user.preferences.po_token = po_token
+    user.preferences.visitor_data = visitor_data
+
+    Invidious::Database::Users.update_preferences(user)
+
+    REDIS_DB.del("invidious:#{user.email}:po_token")
+    REDIS_DB.del("invidious:#{user.email}:visitor_data")
+
+    templated "user/tokens"
+  end
 end

src/invidious/routes/api/manifest.cr

@@ -1,15 +1,4 @@
 module Invidious::Routes::API::Manifest
-  @@proxy_alive : Bool = false
-
-  def self.check_external_proxy
-    begin
-      response = HTTP::Client.get("#{CONFIG.external_videoplayback_proxy}")
-      @@proxy_alive = response.status_code == 200
-    rescue
-      @@proxy_alive = false
-    end
-  end
-
   # /api/manifest/dash/id/:id
   def self.get_dash_video_id(env)
     env.response.headers.add("Access-Control-Allow-Origin", "*")
@@ -38,36 +27,21 @@ module Invidious::Routes::API::Manifest
         haltf env, status_code: response.status_code
       end
 
-      manifest = response.body
-
-      manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
-        url = baseurl.lchop("<BaseURL>")
-        url = url.rchop("</BaseURL>")
-
-        if local
-          uri = URI.parse(url)
-          if @@proxy_alive
-            url = "#{CONFIG.external_videoplayback_proxy}#{uri.request_target}host/#{uri.host}/"
-          else
-            url = "#{HOST_URL}#{uri.request_target}host/#{uri.host}/"
-          end
-        end
-
+      # Proxy URLs for video playback on invidious.
+      # Other API clients can get the original URLs by omiting `local=true`.
+      manifest = response.body.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
+        url = baseurl.lchop("<BaseURL>").rchop("</BaseURL>")
+        url = HttpServer::Utils.proxy_video_url(url, absolute: true) if local
         "<BaseURL>#{url}</BaseURL>"
       end
 
       return manifest
     end
 
-    adaptive_fmts = video.adaptive_fmts
-
+    # Ditto, only proxify URLs if `local=true` is used
     if local
-      adaptive_fmts.each do |fmt|
-        if @@proxy_alive
-          fmt["url"] = JSON::Any.new("#{CONFIG.external_videoplayback_proxy}#{URI.parse(fmt["url"].as_s).request_target}")
-        else
-          fmt["url"] = JSON::Any.new("#{HOST_URL}#{URI.parse(fmt["url"].as_s).request_target}")
-        end
+      video.adaptive_fmts.each do |fmt|
+        fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s, absolute: true))
       end
     end
 
@@ -81,6 +55,10 @@ module Invidious::Routes::API::Manifest
       end
     end
 
+    audio_streams.reject! do |z|
+      z if z.dig?("audioTrack", "audioIsDefault") == false
+    end
+
     manifest = XML.build(indent: "  ", encoding: "UTF-8") do |xml|
       xml.element("MPD", "xmlns": "urn:mpeg:dash:schema:mpd:2011",
         "profiles": "urn:mpeg:dash:profile:full:2011", minBufferTime: "PT1.5S", type: "static",
@@ -203,8 +181,9 @@ module Invidious::Routes::API::Manifest
     manifest = response.body
 
     if local
-      manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
-        path = URI.parse(match).path
+      manifest = manifest.gsub(/^https:\/\/[^\n"]*/m) do |match|
+        uri = URI.parse(match)
+        path = uri.path
 
         path = path.lchop("/videoplayback/")
         path = path.rchop("/")
@@ -233,11 +212,17 @@ module Invidious::Routes::API::Manifest
           raw_params["fvip"] = fvip["fvip"]
         end
 
-        raw_params["local"] = "true"
+        raw_params["host"] = uri.host.not_nil!
 
+        proxy = Invidious::HttpServer::Utils.get_external_proxy
+
+        if !proxy.empty?
+          "#{proxy}/videoplayback?#{raw_params}"
+        else
           "#{HOST_URL}/videoplayback?#{raw_params}"
         end
       end
+    end
 
     manifest
   end

src/invidious/routes/api/v1/videos.cr

@@ -263,60 +263,61 @@ module Invidious::Routes::API::V1::Videos
 
     annotations = ""
 
-    case source
-    when "archive"
-      if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id))
-        annotations = cached_annotation.annotations
-      else
-        index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0')
+    # case source
+    # when "archive"
+    #   if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id))
+    #     annotations = cached_annotation.annotations
+    #   else
+    #     index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0')
 
-        # IA doesn't handle leading hyphens,
-        # so we use https://archive.org/details/youtubeannotations_64
-        if index == "62"
-          index = "64"
-          id = id.sub(/^-/, 'A')
-        end
+    #     # IA doesn't handle leading hyphens,
+    #     # so we use https://archive.org/details/youtubeannotations_64
+    #     if index == "62"
+    #       index = "64"
+    #       id = id.sub(/^-/, 'A')
+    #     end
 
-        file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml")
+    #     file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml")
 
-        location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}"))
+    #     location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}"))
 
-        if !location.headers["Location"]?
-          env.response.status_code = location.status_code
-        end
+    #     if !location.headers["Location"]?
+    #       env.response.status_code = location.status_code
+    #     end
 
-        response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"]))
+    #     response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"]))
 
-        if response.body.empty?
-          haltf env, 404
-        end
+    #     if response.body.empty?
+    #       haltf env, 404
+    #     end
 
-        if response.status_code != 200
-          haltf env, response.status_code
-        end
+    #     if response.status_code != 200
+    #       haltf env, response.status_code
+    #     end
 
-        annotations = response.body
+    #     annotations = response.body
 
-        cache_annotation(id, annotations)
-      end
-    else # "youtube"
-      response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")
+    #     cache_annotation(id, annotations)
+    #   end
+    # else # "youtube"
+    #   response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")
 
-      if response.status_code != 200
-        haltf env, response.status_code
-      end
+    #   if response.status_code != 200
+    #     haltf env, response.status_code
+    #   end
 
-      annotations = response.body
-    end
+    #   annotations = response.body
+    # end
 
-    etag = sha256(annotations)[0, 16]
-    if env.request.headers["If-None-Match"]?.try &.== etag
-      haltf env, 304
-    else
-      env.response.headers["ETag"] = etag
+    # etag = sha256(annotations)[0, 16]
+    # if env.request.headers["If-None-Match"]?.try &.== etag
+    #   haltf env, 304
+    # else
+    #   env.response.headers["ETag"] = etag
+    #   annotations
+    # end
     annotations
   end
-  end
 
   def self.comments(env)
     locale = env.get("preferences").as(Preferences).locale

src/invidious/routes/before_all.cr

@@ -28,12 +28,6 @@ module Invidious::Routes::BeforeAll
       extra_media_csp = ""
     end
 
-    if CONFIG.external_videoplayback_proxy
-      external_videoplayback_proxy = " #{CONFIG.external_videoplayback_proxy}"
-    else
-      external_videoplayback_proxy = ""
-    end
-
     # Only allow the pages at /embed/* to be embedded
     if env.request.resource.starts_with?("/embed")
       frame_ancestors = "'self' file: http: https:"
@@ -49,9 +43,9 @@ module Invidious::Routes::BeforeAll
       "style-src 'self' 'unsafe-inline'",
       "img-src 'self' data:",
       "font-src 'self' data:",
-      "connect-src 'self'" + external_videoplayback_proxy,
+      "connect-src 'self'" + EXT_VIDEOP_LIST,
       "manifest-src 'self'",
-      "media-src 'self' blob:" + extra_media_csp,
+      "media-src 'self' blob:" + extra_media_csp + EXT_VIDEOP_LIST,
       "child-src 'self' blob:",
       "frame-src 'self'",
       "frame-ancestors " + frame_ancestors,

src/invidious/routes/embed.cr

@@ -157,10 +157,12 @@ module Invidious::Routes::Embed
     adaptive_fmts = video.adaptive_fmts
 
     if params.local
-      fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
-      adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+      fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
     end
 
+    # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
+    adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+
     video_streams = video.video_streams
     audio_streams = video.audio_streams
 

src/invidious/routes/feeds.cr

@@ -450,8 +450,8 @@ module Invidious::Routes::Feeds
         if was_insert
           if CONFIG.enable_user_notifications
             Invidious::Database::Users.add_notification(video)
-          else
-            Invidious::Database::Users.feed_needs_update(video)
+            #   else
+            #     Invidious::Database::Users.feed_needs_update(video)
           end
         end
       end

src/invidious/routes/images.cr

@@ -11,29 +11,9 @@ module Invidious::Routes::Images
       end
     end
 
-    # We're encapsulating this into a proc in order to easily reuse this
-    # portion of the code for each request block below.
-    request_proc = ->(response : HTTP::Client::Response) {
-      env.response.status_code = response.status_code
-      response.headers.each do |key, value|
-        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
-          env.response.headers[key] = value
-        end
-      end
-
-      env.response.headers["Access-Control-Allow-Origin"] = "*"
-
-      if response.status_code >= 300
-        env.response.headers.delete("Transfer-Encoding")
-        return
-      end
-
-      proxy_file(response, env)
-    }
-
     begin
-      HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
-        return request_proc.call(resp)
+      GGPHT_POOL.client &.get(url, headers) do |resp|
+        return self.proxy_image(env, resp)
       end
     rescue ex
     end
@@ -61,27 +41,10 @@ module Invidious::Routes::Images
       end
     end
 
-    request_proc = ->(response : HTTP::Client::Response) {
-      env.response.status_code = response.status_code
-      response.headers.each do |key, value|
-        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
-          env.response.headers[key] = value
-        end
-      end
-
-      env.response.headers["Connection"] = "close"
-      env.response.headers["Access-Control-Allow-Origin"] = "*"
-
-      if response.status_code >= 300
-        return env.response.headers.delete("Transfer-Encoding")
-      end
-
-      proxy_file(response, env)
-    }
-
     begin
-      HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
-        return request_proc.call(resp)
+      get_ytimg_pool(authority).client &.get(url, headers) do |resp|
+        env.response.headers["Connection"] = "close"
+        return self.proxy_image(env, resp)
       end
     rescue ex
     end
@@ -101,26 +64,9 @@ module Invidious::Routes::Images
       end
     end
 
-    request_proc = ->(response : HTTP::Client::Response) {
-      env.response.status_code = response.status_code
-      response.headers.each do |key, value|
-        if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
-          env.response.headers[key] = value
-        end
-      end
-
-      env.response.headers["Access-Control-Allow-Origin"] = "*"
-
-      if response.status_code >= 300 && response.status_code != 404
-        return env.response.headers.delete("Transfer-Encoding")
-      end
-
-      proxy_file(response, env)
-    }
-
     begin
-      HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
-        return request_proc.call(resp)
+      get_ytimg_pool("i9").client &.get(url, headers) do |resp|
+        return self.proxy_image(env, resp)
       end
     rescue ex
     end
@@ -165,8 +111,7 @@ module Invidious::Routes::Images
     if name == "maxres.jpg"
       build_thumbnails(id).each do |thumb|
         thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
-        # This can likely be optimized into a (small) pool sometime in the future.
-        if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
+        if get_ytimg_pool("i9").client &.head(thumbnail_resource_path, headers).status_code == 200
           name = thumb[:url] + ".jpg"
           break
         end
@@ -181,7 +126,15 @@ module Invidious::Routes::Images
       end
     end
 
-    request_proc = ->(response : HTTP::Client::Response) {
+    begin
+      get_ytimg_pool("i").client &.get(url, headers) do |resp|
+        return self.proxy_image(env, resp)
+      end
+    rescue ex
+    end
+  end
+
+  private def self.proxy_image(env, response)
     env.response.status_code = response.status_code
     response.headers.each do |key, value|
       if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
@@ -191,19 +144,10 @@ module Invidious::Routes::Images
 
     env.response.headers["Access-Control-Allow-Origin"] = "*"
 
-      if response.status_code >= 300 && response.status_code != 404
+    if response.status_code >= 300
       return env.response.headers.delete("Transfer-Encoding")
     end
 
-      proxy_file(response, env)
-    }
-
-    begin
-      # This can likely be optimized into a (small) pool sometime in the future.
-      HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
-        return request_proc.call(resp)
-      end
-    rescue ex
-    end
+    return proxy_file(response, env)
   end
 end

src/invidious/routes/preferences.cr

@@ -86,6 +86,12 @@ module Invidious::Routes::PreferencesRoute
     show_nick ||= "off"
     show_nick = show_nick == "on"
 
+    po_token = env.params.body["po_token"]?.try &.as(String)
+    po_token ||= CONFIG.default_user_preferences.po_token
+
+    visitor_data = env.params.body["visitor_data"]?.try &.as(String)
+    visitor_data ||= CONFIG.default_user_preferences.visitor_data
+
     comments = [] of String
     2.times do |i|
       comments << (env.params.body["comments[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.comments[i])
@@ -180,6 +186,8 @@ module Invidious::Routes::PreferencesRoute
       vr_mode:                     vr_mode,
       show_nick:                   show_nick,
       save_player_pos:             save_player_pos,
+      po_token:                    po_token,
+      visitor_data:                visitor_data,
     }.to_json)
 
     if user = env.get? "user"

src/invidious/routes/video_playback.cr

@@ -3,6 +3,11 @@ module Invidious::Routes::VideoPlayback
   def self.get_video_playback(env)
     locale = env.get("preferences").as(Preferences).locale
     query_params = env.params.query
+    array = UInt8[0x78, 0]
+    protobuf = Bytes.new(array.size)
+    array.each_with_index do |byte, index|
+      protobuf[index] = byte
+    end
 
     fvip = query_params["fvip"]? || "3"
     mns = query_params["mn"]?.try &.split(",")
@@ -44,7 +49,7 @@ module Invidious::Routes::VideoPlayback
 
     headers["Origin"] = "https://www.youtube.com"
     headers["Referer"] = "https://www.youtube.com/"
-		headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64; rv:122.0) Gecko/20100101 Firefox/122.0"
+    headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
 
     client = make_client(URI.parse(host), region, force_resolve = true)
     response = HTTP::Client::Response.new(500)
@@ -100,7 +105,7 @@ module Invidious::Routes::VideoPlayback
       end
 
       begin
-        client.get(url, headers) do |resp|
+        client.post(url, headers, protobuf) do |resp|
           resp.headers.each do |key, value|
             if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
               env.response.headers[key] = value
@@ -151,7 +156,7 @@ module Invidious::Routes::VideoPlayback
         headers["Range"] = "bytes=#{chunk_start}-#{chunk_end}"
 
         begin
-          client.get(url, headers) do |resp|
+          client.post(url, headers, protobuf) do |resp|
             if first_chunk
               if !env.request.headers["Range"]? && resp.status_code == 206
                 env.response.status_code = 200
@@ -298,7 +303,16 @@ module Invidious::Routes::VideoPlayback
     end
 
     if local
+      external_proxy = Invidious::HttpServer::Utils.get_external_proxy
+      if !external_proxy.empty?
+        url = URI.parse(url)
+        external_proxy = URI.parse(external_proxy)
+        url.host = external_proxy.host
+        url.port = external_proxy.port
+        url = url.to_s
+      else
         url = URI.parse(url).request_target.not_nil!
+      end
       url += "&title=#{URI.encode_www_form(title, space_to_plus: false)}" if title
     end
 

src/invidious/routes/watch.cr

@@ -3,6 +3,14 @@
 module Invidious::Routes::Watch
   def self.handle(env)
     locale = env.get("preferences").as(Preferences).locale
+    if !CONFIG.ignore_user_tokens
+      user_po_token = env.get("preferences").as(Preferences).po_token
+      user_visitor_data = env.get("preferences").as(Preferences).visitor_data
+    else
+      user_po_token = ""
+      user_visitor_data = ""
+    end
+
     region = env.params.query["region"]?
 
     if env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+")
@@ -52,7 +60,7 @@ module Invidious::Routes::Watch
     env.params.query.delete_all("listen")
 
     begin
-      video = get_video(id, region: params.region)
+      video = get_video(id, region: params.region, po_token: user_po_token, visitor_data: user_visitor_data)
     rescue ex : NotFoundException
       LOGGER.error("get_video not found: #{id} : #{ex.message}")
       return error_template(404, ex)
@@ -128,10 +136,12 @@ module Invidious::Routes::Watch
     end
 
     if params.local
-      fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
-      adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+      fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
     end
 
+    # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
+    adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+
     video_streams = video.video_streams
     audio_streams = video.audio_streams
 
@@ -142,6 +152,10 @@ module Invidious::Routes::Watch
       end
     end
 
+    audio_streams.reject! do |z|
+      z if z.dig?("audioTrack", "audioIsDefault") == false
+    end
+
     # Older videos may not have audio sources available.
     # We redirect here so they're not unplayable
     if audio_streams.empty? && !video.live_now

src/invidious/routing.cr

@@ -77,6 +77,7 @@ module Invidious::Routing
     post "/authorize_token", Routes::Account, :post_authorize_token
     get "/token_manager", Routes::Account, :token_manager
     post "/token_ajax", Routes::Account, :token_ajax
+    get "/generate_tokens", Routes::Account, :generate_tokens
     post "/subscription_ajax", Routes::Subscriptions, :toggle_subscription
     get "/subscription_manager", Routes::Subscriptions, :subscription_manager
   end

src/invidious/user/preferences.cr

@@ -57,6 +57,10 @@ struct Preferences
   property volume : Int32 = CONFIG.default_user_preferences.volume
   property save_player_pos : Bool = CONFIG.default_user_preferences.save_player_pos
 
+  @[YAML::Field(converter: Preferences::ProcessString)]
+  property po_token : String = ""
+  property visitor_data : String = ""
+
   module BoolToString
     def self.to_json(value : String, json : JSON::Builder)
       json.string value

src/invidious/videos.cr

@@ -294,7 +294,7 @@ struct Video
   predicate_bool upcoming, isUpcoming
 end
 
-def get_video(id, refresh = true, region = nil, force_refresh = false)
+def get_video(id, refresh = true, region = nil, force_refresh = false, po_token = "", visitor_data = "")
   if (video = Invidious::Database::Videos.select(id)) && !region
     # If record was last updated over 10 minutes ago, or video has since premiered,
     # refresh (expire param in response lasts for 6 hours)
@@ -304,7 +304,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
        force_refresh ||
        video.schema_version != Video::SCHEMA_VERSION # cache control
       begin
-        video = fetch_video(id, region)
+        video = fetch_video(id, region, po_token, visitor_data)
         Invidious::Database::Videos.insert(video)
       rescue ex
         Invidious::Database::Videos.delete(id)
@@ -312,7 +312,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
       end
     end
   else
-    video = fetch_video(id, region)
+    video = fetch_video(id, region, po_token, visitor_data)
     Invidious::Database::Videos.insert(video) if !region
   end
 
@@ -320,11 +320,11 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
 rescue DB::Error
   # Avoid common `DB::PoolRetryAttemptsExceeded` error and friends
   # Note: All DB errors inherit from `DB::Error`
-  return fetch_video(id, region)
+  return fetch_video(id, region, po_token, visitor_data)
 end
 
-def fetch_video(id, region)
-  info = extract_video_info(video_id: id)
+def fetch_video(id, region, po_token, visitor_data)
+  info = extract_video_info(video_id: id, user_po_token: po_token, user_visitor_data: visitor_data)
 
   if reason = info["reason"]?
     if reason == "Video unavailable"

src/invidious/videos/caption.cr

@@ -123,6 +123,7 @@ module Invidious::Videos
       "Esperanto",
       "Estonian",
       "Filipino",
+      "Filipino (auto-generated)",
       "Finnish",
       "French",
       "French (auto-generated)",

src/invidious/videos/parser.cr

@@ -50,12 +50,17 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
   }
 end
 
-def extract_video_info(video_id : String)
+def extract_video_info(video_id : String, user_po_token, user_visitor_data)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
 
+  redis_po_token, redis_visitor_data = Tokens.get_tokens
+
+  po_token = (user_po_token if !user_po_token.empty?) || redis_po_token || CONFIG.po_token
+  visitor_data = (user_visitor_data if !user_visitor_data.empty?) || redis_visitor_data || CONFIG.visitor_data
+
   # Fetch data from the player endpoint
-  player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
+  player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config, po_token: po_token, visitor_data: visitor_data)
 
   playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
 
@@ -104,22 +109,22 @@ def extract_video_info(video_id : String)
 
   # Don't use Android client if po_token is passed because po_token doesn't
   # work for Android client.
-  if reason.nil? && CONFIG.po_token.nil?
+  if reason.nil? && po_token.nil?
     # Fetch the video streams using an Android client in order to get the
     # decrypted URLs and maybe fix throttling issues (#2194). See the
     # following issue for an explanation about decrypted URLs:
     # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
     client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
-    new_player_response = try_fetch_streaming_data(video_id, client_config)
+    new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data)
   end
 
   # Last hope
   # Only trigger if reason found and po_token or didn't work wth Android client.
   # TvHtml5ScreenEmbed now requires sig helper for it to work but po_token is not required
   # if the IP address is not blocked.
-  if CONFIG.po_token && reason || CONFIG.po_token.nil? && new_player_response.nil?
+  if po_token.nil? && reason || po_token.nil? && new_player_response.nil?
     client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
-    new_player_response = try_fetch_streaming_data(video_id, client_config)
+    new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data)
   end
 
   # Replace player response and reset reason
@@ -140,7 +145,7 @@ def extract_video_info(video_id : String)
   if streaming_data = player_response["streamingData"]?
     %w[formats adaptiveFormats].each do |key|
       streaming_data.as_h[key]?.try &.as_a.each do |format|
-        format.as_h["url"] = JSON::Any.new(convert_url(format))
+        format.as_h["url"] = JSON::Any.new(convert_url(format, po_token))
       end
     end
 
@@ -153,9 +158,9 @@ def extract_video_info(video_id : String)
   return params
 end
 
-def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
+def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig, po_token, visitor_data) : Hash(String, JSON::Any)?
   LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
-  response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)
+  response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config, po_token: po_token, visitor_data: visitor_data)
 
   playability_status = response["playabilityStatus"]["status"]
   LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
@@ -455,7 +460,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   return params
 end
 
-private def convert_url(fmt)
+private def convert_url(fmt, po_token)
   if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
     sp = cfr["sp"]
     url = URI.parse(cfr["url"])
@@ -473,7 +478,9 @@ private def convert_url(fmt)
   n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
   params["n"] = n if n
 
-  if token = CONFIG.po_token
+  if !po_token.nil?
+    params["pot"] = po_token
+  elsif token = CONFIG.po_token
     params["pot"] = token
   end
 

src/invidious/views/user/preferences.ecr

@@ -126,6 +126,24 @@
                 <input name="save_player_pos" id="save_player_pos" type="checkbox" <% if preferences.save_player_pos %>checked<% end %>>
             </div>
 
+            <% if !CONFIG.ignore_user_tokens %>
+			<div class="pure-control-group">
+                <label for="po_token"><%= translate(locale, "preferences_po_token") %></label>
+                <input name="po_token" id="po_token" type="text" value="<%= preferences.po_token %>">
+            </div>
+
+			<div class="pure-control-group">
+                <label for="visitor_data"><%= translate(locale, "preferences_visitor_data") %></label>
+                <input name="visitor_data" id="visitor_data" type="text" value="<%= preferences.visitor_data %>">
+            </div>
+
+        <% if env.get?("user") %>
+			<div class="pure-control-group">
+				<a href="/generate_tokens?referer=<%= URI.encode_www_form(referer) %>"><%= translate(locale, "Generate po_token and visitor_data for your account") %></a>
+			</div>
+        <% end %>
+        <% end %>
+
             <legend><%= translate(locale, "preferences_category_visual") %></legend>
 
             <div class="pure-control-group">

src/invidious/views/user/tokens.ecr

@@ -0,0 +1,15 @@
+<% content_for "header" do %>
+<title><%= translate(locale, "Invidious token generator") %> - Invidious</title>
+<% end %>
+
+<div class="pure-g">
+    <div class="pure-u-1 pure-u-lg-1-5"></div>
+    <div class="pure-u-1 pure-u-lg-3-5">
+        <div class="h-box">
+		<p>po_token and visitor_data successfully generated!</p>
+		<p>po_token: <%= po_token %></p>
+		<p>visitor_data: <%= visitor_data %></p>
+        </div>
+    </div>
+    <div class="pure-u-1 pure-u-lg-1-5"></div>
+</div>

src/invidious/yt_backend/connection_pool.cr

@@ -1,17 +1,6 @@
-def add_yt_headers(request)
-  request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
-  request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
-
-  request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
-  request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
-  request.headers["Accept-Language"] ||= "en-us,en;q=0.5"
-
-  # Preserve original cookies and add new YT consent cookie for EU servers
-  request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}"
-  if !CONFIG.cookies.empty?
-    request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
-  end
-end
+# Mapping of subdomain => YoutubeConnectionPool
+# This is needed as we may need to access arbitrary subdomains of ytimg
+private YTIMG_POOLS = {} of String => YoutubeConnectionPool
 
 struct YoutubeConnectionPool
   property! url : URI
@@ -26,12 +15,16 @@ struct YoutubeConnectionPool
 
   def client(&)
     conn = pool.checkout
+    # Proxy needs to be reinstated every time we get a client from the pool
+    conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+
     begin
       response = yield conn
     rescue ex
       conn.close
-      conn = HTTP::Client.new(url)
 
+      conn = HTTP::Client.new(url)
+      conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
       conn.family = CONFIG.force_resolve
       conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
       conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
@@ -54,6 +47,21 @@ struct YoutubeConnectionPool
   end
 end
 
+def add_yt_headers(request)
+  request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
+  request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
+
+  request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
+  request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+  request.headers["Accept-Language"] ||= "en-us,en;q=0.5"
+
+  # Preserve original cookies and add new YT consent cookie for EU servers
+  request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}"
+  if !CONFIG.cookies.empty?
+    request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
+  end
+end
+
 def make_client(url : URI, region = nil, force_resolve : Bool = false)
   client = HTTP::Client.new(url)
 
@@ -77,3 +85,18 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &)
     client.close
   end
 end
+
+# Fetches a HTTP pool for the specified subdomain of ytimg.com
+#
+# Creates a new one when the specified pool for the subdomain does not exist
+def get_ytimg_pool(subdomain)
+  if pool = YTIMG_POOLS[subdomain]?
+    return pool
+  else
+    LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"")
+    pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size)
+    YTIMG_POOLS[subdomain] = pool
+
+    return pool
+  end
+end

src/invidious/yt_backend/youtube_api.cr

@@ -3,6 +3,8 @@
 #
 
 module YoutubeAPI
+  @@visitor_data : String = ""
+
   extend self
 
   # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history
@@ -320,7 +322,9 @@ module YoutubeAPI
       client_context["client"]["platform"] = platform
     end
 
-    if CONFIG.visitor_data.is_a?(String)
+    if !@@visitor_data.empty?
+      client_context["client"]["visitorData"] = @@visitor_data
+    elsif CONFIG.visitor_data.is_a?(String)
       client_context["client"]["visitorData"] = CONFIG.visitor_data.as(String)
     end
 
@@ -455,8 +459,13 @@ module YoutubeAPI
     video_id : String,
     *, # Force the following parameters to be passed by name
     params : String,
-    client_config : ClientConfig | Nil = nil
+    client_config : ClientConfig | Nil = nil,
+    po_token : String | Nil,
+    visitor_data : String | Nil,
   )
+    if visitor_data
+      @@visitor_data = visitor_data
+    end
     # Playback context, separate because it can be different between clients
     playback_ctx = {
       "html5Preference" => "HTML5_PREF_WANTS",
@@ -482,7 +491,7 @@ module YoutubeAPI
         "contentPlaybackContext" => playback_ctx,
       },
       "serviceIntegrityDimensions" => {
-        "poToken" => CONFIG.po_token,
+        "poToken" => po_token || CONFIG.po_token,
       },
     }
 
@@ -596,7 +605,7 @@ module YoutubeAPI
   def _post_json(
     endpoint : String,
     data : Hash,
-    client_config : ClientConfig | Nil
+    client_config : ClientConfig | Nil,
   ) : Hash(String, JSON::Any)
     # Use the default client config if nil is passed
     client_config ||= DEFAULT_CLIENT_CONFIG
@@ -616,7 +625,9 @@ module YoutubeAPI
       headers["User-Agent"] = user_agent
     end
 
-    if CONFIG.visitor_data.is_a?(String)
+    if !@@visitor_data.empty?
+      headers["X-Goog-Visitor-Id"] = @@visitor_data
+    elsif CONFIG.visitor_data.is_a?(String)
       headers["X-Goog-Visitor-Id"] = CONFIG.visitor_data.as(String)
     end