Tokens: Option to disable user tokens.

#18

.forgejo/workflows/ci.yml

@@ -8,6 +8,8 @@ on:
   push:
     branches:
       - "master"
+      - "experimental"
+      - "experimental2"
 
 jobs:
   build:

crystal_formatters.py

@@ -0,0 +1,62 @@
+import lldb
+
+class CrystalArraySyntheticProvider:
+    def __init__(self, valobj, internal_dict):
+        self.valobj = valobj
+        self.buffer = None
+        self.size = 0
+
+    def update(self):
+        if self.valobj.type.is_pointer:
+            self.valobj = self.valobj.Dereference()
+        self.size = int(self.valobj.child[0].value)
+        self.type = self.valobj.type
+        self.buffer = self.valobj.child[3]
+
+    def num_children(self):
+        size = 0 if self.size is None else self.size
+        return size
+
+    def get_child_index(self, name):
+        try:
+            return int(name.lstrip('[').rstrip(']'))
+        except:
+            return -1
+
+    def get_child_at_index(self,index):
+        if index >= self.size:
+            return None
+        try:
+            elementType = self.buffer.type.GetPointeeType()
+            offset = elementType.size * index
+            return self.buffer.CreateChildAtOffset('[' + str(index) + ']', offset, elementType)
+        except Exception as e:
+            print('Got exception %s' % (str(e)))
+            return None
+
+def findType(name, module):
+    cachedTypes = module.GetTypes()
+    for idx in range(cachedTypes.GetSize()):
+        type = cachedTypes.GetTypeAtIndex(idx)
+        if type.name == name:
+            return type
+    return None
+
+
+def CrystalString_SummaryProvider(value, dict):
+    error = lldb.SBError()
+    if value.TypeIsPointerType():
+        value = value.Dereference()
+    process = value.GetTarget().GetProcess()
+    byteSize = int(value.child[0].value)
+    len = int(value.child[1].value)
+    len = byteSize or len
+    strAddr = value.child[2].load_addr
+    val = process.ReadCStringFromMemory(strAddr, len + 1, error)
+    return '"%s"' % val
+
+
+def __lldb_init_module(debugger, dict):
+    debugger.HandleCommand(r'type synthetic add -l crystal_formatters.CrystalArraySyntheticProvider -x "^Array\(.+\)(\s*\**)?" -w Crystal')
+    debugger.HandleCommand(r'type summary add -F crystal_formatters.CrystalString_SummaryProvider -x "^(String|\(String \| Nil\))(\s*\**)?$" -w Crystal')
+    debugger.HandleCommand(r'type category enable Crystal')

src/invidious.cr

@@ -67,11 +67,13 @@ REDIS_DB = Redis::PooledClient.new(unixsocket: CONFIG.redis_socket || nil, url:
 if REDIS_DB.ping
   puts "Connected to redis"
 end
-ARCHIVE_URL = URI.parse("https://archive.org")
-PUBSUB_URL  = URI.parse("https://pubsubhubbub.appspot.com")
-REDDIT_URL  = URI.parse("https://www.reddit.com")
-YT_URL      = URI.parse("https://www.youtube.com")
-HOST_URL    = make_host_url(Kemal.config)
+ARCHIVE_URL     = URI.parse("https://archive.org")
+PUBSUB_URL      = URI.parse("https://pubsubhubbub.appspot.com")
+REDDIT_URL      = URI.parse("https://www.reddit.com")
+YT_URL          = URI.parse("https://www.youtube.com")
+PUBSUB_HOST_URL = CONFIG.pubsub_domain
+HOST_URL        = make_host_url(Kemal.config)
+EXT_VIDEOP_LIST = gen_videoplayback_proxy_list()
 
 CHARS_SAFE         = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
 TEST_IDS           = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
@@ -188,10 +190,14 @@ Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new
 
 Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new
 
-if CONFIG.external_videoplayback_proxy
+if !CONFIG.external_videoplayback_proxy.empty?
   Invidious::Jobs.register Invidious::Jobs::CheckExternalProxy.new
 end
 
+if CONFIG.refresh_tokens
+  Invidious::Jobs.register Invidious::Jobs::RefreshTokens.new
+end
+
 Invidious::Jobs.start_all
 
 def popular_videos

src/invidious/channels/channels.cr

@@ -251,8 +251,8 @@ def fetch_channel(ucid, pull_all_videos : Bool)
       LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
       if CONFIG.enable_user_notifications
         Invidious::Database::Users.add_notification(video)
-      else
-        Invidious::Database::Users.feed_needs_update(video)
+        #   else
+        #     Invidious::Database::Users.feed_needs_update(video)
       end
     else
       LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
@@ -287,8 +287,8 @@ def fetch_channel(ucid, pull_all_videos : Bool)
           if was_insert
             if CONFIG.enable_user_notifications
               Invidious::Database::Users.add_notification(video)
-            else
-              Invidious::Database::Users.feed_needs_update(video)
+              # else
+              #   Invidious::Database::Users.feed_needs_update(video)
             end
           end
         end

src/invidious/config.cr

@@ -45,6 +45,8 @@ struct ConfigPreferences
   property vr_mode : Bool = true
   property show_nick : Bool = true
   property save_player_pos : Bool = false
+  property po_token : String = ""
+  property visitor_data : String = ""
 
   def to_tuple
     {% begin %}
@@ -87,6 +89,8 @@ class Config
 
   # Used to tell Invidious it is behind a proxy, so links to resources should be https://
   property https_only : Bool?
+  # Enable or disable CSP
+  property csp : Bool? = true
   # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
   property hmac_key : String = ""
   # Domain to be used for links to resources on the site where an absolute URL is required
@@ -176,7 +180,16 @@ class Config
   # of the backend
   property backends_delimiter : String = "|"
 
-  property external_videoplayback_proxy : String?
+  # External videoplayback proxies list. They should include `https://`
+  # at the start of the URI
+  property external_videoplayback_proxy : Array(String) = [] of String
+
+  # Job to refresh tokens from a Redis compatible DB
+  property refresh_tokens : Bool = true
+
+  property pubsub_domain : String = ""
+
+  property ignore_user_tokens : Bool = false
 
   # Materialious redirects
   property materialious_domain : String?

src/invidious/database/users.cr

@@ -154,15 +154,17 @@ module Invidious::Database::Users
   #  Update (misc)
   # -------------------
 
-  def feed_needs_update(video : ChannelVideo)
-    request = <<-SQL
-      UPDATE users
-      SET feed_needs_update = true
-      WHERE $1 = ANY(subscriptions)
-    SQL
+  # Feeds never need update. PubSubHubBub is the one that sends videos to
+  # invidious.
+  #   def feed_needs_update(video : ChannelVideo)
+  #     request = <<-SQL
+  #       UPDATE users
+  #       SET feed_needs_update = true
+  #       WHERE $1 = ANY(subscriptions)
+  #     SQL
 
-    PG_DB.exec(request, video.ucid)
-  end
+  #     PG_DB.exec(request, video.ucid)
+  #   end
 
   def update_preferences(user : User)
     request = <<-SQL

src/invidious/database/videos.cr

@@ -10,8 +10,8 @@ module Invidious::Database::Videos
       ON CONFLICT (id) DO NOTHING
     SQL
 
-    REDIS_DB.set(video.id, video.info.to_json, ex: 3600)
-    REDIS_DB.set(video.id + ":time", video.updated, ex: 3600)
+    REDIS_DB.set(video.id, video.info.to_json, ex: 14400)
+    REDIS_DB.set(video.id + ":time", video.updated, ex: 14400)
   end
 
   def delete(id)

src/invidious/helpers/redis_tokens.cr

@@ -0,0 +1,49 @@
+module Tokens
+  extend self
+  @@po_token : String | Nil
+  @@visitor_data : String | Nil
+
+  def refresh_tokens
+    @@po_token = REDIS_DB.get("invidious:po_token")
+    @@visitor_data = REDIS_DB.get("invidious:visitor_data")
+    LOGGER.debug("RefreshTokens: Tokens are:")
+    LOGGER.debug("RefreshTokens: po_token: #{@@po_token}")
+    LOGGER.debug("RefreshTokens: visitor_data: #{@@visitor_data}")
+  end
+
+  def get_tokens
+    return {@@po_token, @@visitor_data}
+  end
+
+  def get_po_token
+    return @@po_token
+  end
+
+  def get_visitor_data
+    return @@visitor_data
+  end
+
+  def generate_tokens(user : String)
+    po_token = ""
+    visitor_data = ""
+    attempts = 0
+
+    LOGGER.debug("Generating po_token and visitor_data for user: '#{user}'")
+    REDIS_DB.publish("generate-token", "#{user}")
+
+    while REDIS_DB.get("invidious:#{user}:po_token").nil? && REDIS_DB.get("invidious:#{user}:visitor_data").nil?
+      if attempts > 50
+        break
+      end
+      LOGGER.debug("Waiting for tokens to arrive at redis for user: '#{user}'")
+      attempts += 1
+      sleep 250.milliseconds
+    end
+
+    po_token = REDIS_DB.get("invidious:#{user}:po_token")
+    visitor_data = REDIS_DB.get("invidious:#{user}:visitor_data")
+
+    LOGGER.debug("Tokens successfully generated for user: '#{user}'")
+    return {po_token, visitor_data}
+  end
+end

src/invidious/helpers/utils.cr

@@ -294,7 +294,7 @@ def subscribe_pubsub(topic, key)
   signature = "#{time}:#{nonce}"
 
   body = {
-    "hub.callback"      => "#{HOST_URL}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}",
+    "hub.callback"      => "#{PUBSUB_HOST_URL}/feed/webhook/v1:#{time}:#{nonce}:#{OpenSSL::HMAC.hexdigest(:sha1, key, signature)}",
     "hub.topic"         => "https://www.youtube.com/xml/feeds/videos.xml?#{topic}",
     "hub.verify"        => "async",
     "hub.mode"          => "subscribe",
@@ -383,3 +383,17 @@ def parse_link_endpoint(endpoint : JSON::Any, text : String, video_id : String)
   end
   return text
 end
+
+# Generates a list of external videoplayback proxies for
+# CSP
+def gen_videoplayback_proxy_list
+  if !CONFIG.external_videoplayback_proxy.empty?
+    external_videoplayback_proxy = ""
+    CONFIG.external_videoplayback_proxy.each do |proxy|
+      external_videoplayback_proxy += " #{proxy}"
+    end
+  else
+    external_videoplayback_proxy = ""
+  end
+  return external_videoplayback_proxy
+end

src/invidious/http_server/utils.cr

@@ -4,6 +4,27 @@ module Invidious::HttpServer
   module Utils
     extend self
 
+    @@proxy_alive : String = ""
+
+    def check_external_proxy
+      CONFIG.external_videoplayback_proxy.each do |proxy|
+        begin
+          response = HTTP::Client.get(proxy)
+          if response.status_code == 200
+            @@proxy_alive = proxy
+            LOGGER.debug("CheckExternalProxy: Proxy set to: '#{proxy}'")
+            break
+          end
+        rescue
+          LOGGER.debug("CheckExternalProxy: Proxy '#{proxy}' is not available")
+        end
+      end
+    end
+
+    def get_external_proxy
+      return @@proxy_alive
+    end
+
     def proxy_video_url(raw_url : String, *, region : String? = nil, absolute : Bool = false)
       url = URI.parse(raw_url)
 
@@ -14,7 +35,11 @@ module Invidious::HttpServer
       url.query_params = params
 
       if absolute
-        return "#{HOST_URL}#{url.request_target}"
+        if !@@proxy_alive.empty?
+          return "#{@@proxy_alive}#{url.request_target}"
+        else
+          return "#{HOST_URL}#{url.request_target}"
+        end
       else
         return url.request_target
       end

src/invidious/jobs/check_external_proxy.cr

@@ -4,7 +4,7 @@ class Invidious::Jobs::CheckExternalProxy < Invidious::Jobs::BaseJob
 
   def begin
     loop do
-      Invidious::Routes::API::Manifest.check_external_proxy
+      HttpServer::Utils.check_external_proxy
       LOGGER.info("CheckExternalProxy: Done, sleeping for 1 minute")
       sleep 1.minutes
       Fiber.yield

src/invidious/jobs/refresh_tokens.cr

@@ -0,0 +1,13 @@
+class Invidious::Jobs::RefreshTokens < Invidious::Jobs::BaseJob
+  def initialize
+  end
+
+  def begin
+    loop do
+      Tokens.refresh_tokens
+      LOGGER.info("RefreshTokens: Done, sleeping for 5 seconds")
+      sleep 5.seconds
+      Fiber.yield
+    end
+  end
+end

src/invidious/jobs/subscribe_to_feeds_job.cr

@@ -30,6 +30,8 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
           spawn do
             begin
               response = subscribe_pubsub(ucid, hmac_key)
+              LOGGER.debug("SubscribeToFeedsJob: Subscribed to #{ucid}.")
+              LOGGER.trace("SubscribeToFeedsJob: response.body: #{response.body}")
 
               if response.status_code >= 400
                 LOGGER.error("SubscribeToFeedsJob: #{ucid} : #{response.body}")

src/invidious/routes/account.cr

@@ -349,4 +349,40 @@ module Invidious::Routes::Account
       return "{}"
     end
   end
+
+  # -------------------
+  #  poToken and visitorData tokens generation
+  # -------------------
+
+  # Generates a poToken & visitorData for the user, server side
+  def generate_tokens(env)
+    locale = env.get("preferences").as(Preferences).locale
+    preferences = env.get("preferences").as(Preferences)
+
+    user = env.get? "user"
+    sid = env.get? "sid"
+    referer = get_referer(env)
+
+    if !user
+      return env.redirect referer
+    end
+
+    user = user.as(User)
+
+    po_token, visitor_data = Tokens.generate_tokens(user.email)
+
+    if po_token.nil? || visitor_data.nil?
+      return error_template(500, "Internal server error. Please submit an issue here IF THE ISSUE PERSISTS: https://git.nadeko.net/Fijxu/invidious/issues")
+    end
+
+    user.preferences.po_token = po_token
+    user.preferences.visitor_data = visitor_data
+
+    Invidious::Database::Users.update_preferences(user)
+
+    REDIS_DB.del("invidious:#{user.email}:po_token")
+    REDIS_DB.del("invidious:#{user.email}:visitor_data")
+
+    templated "user/tokens"
+  end
 end

src/invidious/routes/api/manifest.cr

@@ -1,15 +1,4 @@
 module Invidious::Routes::API::Manifest
-  @@proxy_alive : Bool = false
-
-  def self.check_external_proxy
-    begin
-      response = HTTP::Client.get("#{CONFIG.external_videoplayback_proxy}")
-      @@proxy_alive = response.status_code == 200
-    rescue
-      @@proxy_alive = false
-    end
-  end
-
   # /api/manifest/dash/id/:id
   def self.get_dash_video_id(env)
     env.response.headers.add("Access-Control-Allow-Origin", "*")
@@ -38,36 +27,21 @@ module Invidious::Routes::API::Manifest
         haltf env, status_code: response.status_code
       end
 
-      manifest = response.body
-
-      manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
-        url = baseurl.lchop("<BaseURL>")
-        url = url.rchop("</BaseURL>")
-
-        if local
-          uri = URI.parse(url)
-          if @@proxy_alive
-            url = "#{CONFIG.external_videoplayback_proxy}#{uri.request_target}host/#{uri.host}/"
-          else
-            url = "#{HOST_URL}#{uri.request_target}host/#{uri.host}/"
-          end
-        end
-
+      # Proxy URLs for video playback on invidious.
+      # Other API clients can get the original URLs by omiting `local=true`.
+      manifest = response.body.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
+        url = baseurl.lchop("<BaseURL>").rchop("</BaseURL>")
+        url = HttpServer::Utils.proxy_video_url(url, absolute: true) if local
         "<BaseURL>#{url}</BaseURL>"
       end
 
       return manifest
     end
 
-    adaptive_fmts = video.adaptive_fmts
-
+    # Ditto, only proxify URLs if `local=true` is used
     if local
-      adaptive_fmts.each do |fmt|
-        if @@proxy_alive
-          fmt["url"] = JSON::Any.new("#{CONFIG.external_videoplayback_proxy}#{URI.parse(fmt["url"].as_s).request_target}")
-        else
-          fmt["url"] = JSON::Any.new("#{HOST_URL}#{URI.parse(fmt["url"].as_s).request_target}")
-        end
+      video.adaptive_fmts.each do |fmt|
+        fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s, absolute: true))
       end
     end
 
@@ -203,8 +177,9 @@ module Invidious::Routes::API::Manifest
     manifest = response.body
 
     if local
-      manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
-        path = URI.parse(match).path
+      manifest = manifest.gsub(/^https:\/\/[^\n"]*/m) do |match|
+        uri = URI.parse(match)
+        path = uri.path
 
         path = path.lchop("/videoplayback/")
         path = path.rchop("/")
@@ -233,9 +208,15 @@ module Invidious::Routes::API::Manifest
           raw_params["fvip"] = fvip["fvip"]
         end
 
-        raw_params["local"] = "true"
+        raw_params["host"] = uri.host.not_nil!
 
-        "#{HOST_URL}/videoplayback?#{raw_params}"
+        proxy = Invidious::HttpServer::Utils.get_external_proxy
+
+        if !proxy.empty?
+          "#{proxy}/videoplayback?#{raw_params}"
+        else
+          "#{HOST_URL}/videoplayback?#{raw_params}"
+        end
       end
     end
 

src/invidious/routes/api/v1/videos.cr

@@ -263,59 +263,60 @@ module Invidious::Routes::API::V1::Videos
 
     annotations = ""
 
-    case source
-    when "archive"
-      if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id))
-        annotations = cached_annotation.annotations
-      else
-        index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0')
+    # case source
+    # when "archive"
+    #   if CONFIG.cache_annotations && (cached_annotation = Invidious::Database::Annotations.select(id))
+    #     annotations = cached_annotation.annotations
+    #   else
+    #     index = CHARS_SAFE.index!(id[0]).to_s.rjust(2, '0')
 
-        # IA doesn't handle leading hyphens,
-        # so we use https://archive.org/details/youtubeannotations_64
-        if index == "62"
-          index = "64"
-          id = id.sub(/^-/, 'A')
-        end
+    #     # IA doesn't handle leading hyphens,
+    #     # so we use https://archive.org/details/youtubeannotations_64
+    #     if index == "62"
+    #       index = "64"
+    #       id = id.sub(/^-/, 'A')
+    #     end
 
-        file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml")
+    #     file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml")
 
-        location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}"))
+    #     location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}"))
 
-        if !location.headers["Location"]?
-          env.response.status_code = location.status_code
-        end
+    #     if !location.headers["Location"]?
+    #       env.response.status_code = location.status_code
+    #     end
 
-        response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"]))
+    #     response = make_client(URI.parse(location.headers["Location"]), &.get(location.headers["Location"]))
 
-        if response.body.empty?
-          haltf env, 404
-        end
+    #     if response.body.empty?
+    #       haltf env, 404
+    #     end
 
-        if response.status_code != 200
-          haltf env, response.status_code
-        end
+    #     if response.status_code != 200
+    #       haltf env, response.status_code
+    #     end
 
-        annotations = response.body
+    #     annotations = response.body
 
-        cache_annotation(id, annotations)
-      end
-    else # "youtube"
-      response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")
+    #     cache_annotation(id, annotations)
+    #   end
+    # else # "youtube"
+    #   response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")
 
-      if response.status_code != 200
-        haltf env, response.status_code
-      end
+    #   if response.status_code != 200
+    #     haltf env, response.status_code
+    #   end
 
-      annotations = response.body
-    end
+    #   annotations = response.body
+    # end
 
-    etag = sha256(annotations)[0, 16]
-    if env.request.headers["If-None-Match"]?.try &.== etag
-      haltf env, 304
-    else
-      env.response.headers["ETag"] = etag
-      annotations
-    end
+    # etag = sha256(annotations)[0, 16]
+    # if env.request.headers["If-None-Match"]?.try &.== etag
+    #   haltf env, 304
+    # else
+    #   env.response.headers["ETag"] = etag
+    #   annotations
+    # end
+    annotations
   end
 
   def self.comments(env)

src/invidious/routes/before_all.cr

@@ -28,12 +28,6 @@ module Invidious::Routes::BeforeAll
       extra_media_csp = ""
     end
 
-    if CONFIG.external_videoplayback_proxy
-      external_videoplayback_proxy = " #{CONFIG.external_videoplayback_proxy}"
-    else
-      external_videoplayback_proxy = ""
-    end
-
     # Only allow the pages at /embed/* to be embedded
     if env.request.resource.starts_with?("/embed")
       frame_ancestors = "'self' file: http: https:"
@@ -49,9 +43,9 @@ module Invidious::Routes::BeforeAll
       "style-src 'self' 'unsafe-inline'",
       "img-src 'self' data:",
       "font-src 'self' data:",
-      "connect-src 'self'" + external_videoplayback_proxy,
+      "connect-src 'self'" + EXT_VIDEOP_LIST,
       "manifest-src 'self'",
-      "media-src 'self' blob:" + extra_media_csp,
+      "media-src 'self' blob:" + extra_media_csp + EXT_VIDEOP_LIST,
       "child-src 'self' blob:",
       "frame-src 'self'",
       "frame-ancestors " + frame_ancestors,

src/invidious/routes/embed.cr

@@ -157,10 +157,12 @@ module Invidious::Routes::Embed
     adaptive_fmts = video.adaptive_fmts
 
     if params.local
-      fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
-      adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+      fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
     end
 
+    # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
+    adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+
     video_streams = video.video_streams
     audio_streams = video.audio_streams
 

src/invidious/routes/feeds.cr

@@ -450,8 +450,8 @@ module Invidious::Routes::Feeds
         if was_insert
           if CONFIG.enable_user_notifications
             Invidious::Database::Users.add_notification(video)
-          else
-            Invidious::Database::Users.feed_needs_update(video)
+            #   else
+            #     Invidious::Database::Users.feed_needs_update(video)
           end
         end
       end

src/invidious/routes/preferences.cr

@@ -86,6 +86,12 @@ module Invidious::Routes::PreferencesRoute
     show_nick ||= "off"
     show_nick = show_nick == "on"
 
+    po_token = env.params.body["po_token"]?.try &.as(String)
+    po_token ||= CONFIG.default_user_preferences.po_token
+
+    visitor_data = env.params.body["visitor_data"]?.try &.as(String)
+    visitor_data ||= CONFIG.default_user_preferences.visitor_data
+
     comments = [] of String
     2.times do |i|
       comments << (env.params.body["comments[#{i}]"]?.try &.as(String) || CONFIG.default_user_preferences.comments[i])
@@ -180,6 +186,8 @@ module Invidious::Routes::PreferencesRoute
       vr_mode:                     vr_mode,
       show_nick:                   show_nick,
       save_player_pos:             save_player_pos,
+      po_token:                    po_token,
+      visitor_data:                visitor_data,
     }.to_json)
 
     if user = env.get? "user"

src/invidious/routes/video_playback.cr

@@ -42,9 +42,9 @@ module Invidious::Routes::VideoPlayback
       headers["Range"] = "bytes=#{range_for_head}"
     end
 
-		headers["Origin"] = "https://www.youtube.com"
-		headers["Referer"] = "https://www.youtube.com/"
-		headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64; rv:122.0) Gecko/20100101 Firefox/122.0"
+    headers["Origin"] = "https://www.youtube.com"
+    headers["Referer"] = "https://www.youtube.com/"
+    headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
 
     client = make_client(URI.parse(host), region, force_resolve = true)
     response = HTTP::Client::Response.new(500)
@@ -298,7 +298,16 @@ module Invidious::Routes::VideoPlayback
     end
 
     if local
-      url = URI.parse(url).request_target.not_nil!
+      external_proxy = Invidious::HttpServer::Utils.get_external_proxy
+      if !external_proxy.empty?
+        url = URI.parse(url)
+        external_proxy = URI.parse(external_proxy)
+        url.host = external_proxy.host
+        url.port = external_proxy.port
+        url = url.to_s
+      else
+        url = URI.parse(url).request_target.not_nil!
+      end
       url += "&title=#{URI.encode_www_form(title, space_to_plus: false)}" if title
     end
 

src/invidious/routes/watch.cr

@@ -3,6 +3,14 @@
 module Invidious::Routes::Watch
   def self.handle(env)
     locale = env.get("preferences").as(Preferences).locale
+    if !CONFIG.ignore_user_tokens
+      user_po_token = env.get("preferences").as(Preferences).po_token
+      user_visitor_data = env.get("preferences").as(Preferences).visitor_data
+    else
+      user_po_token = ""
+      user_visitor_data = ""
+    end
+
     region = env.params.query["region"]?
 
     if env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+")
@@ -52,7 +60,7 @@ module Invidious::Routes::Watch
     env.params.query.delete_all("listen")
 
     begin
-      video = get_video(id, region: params.region)
+      video = get_video(id, region: params.region, po_token: user_po_token, visitor_data: user_visitor_data)
     rescue ex : NotFoundException
       LOGGER.error("get_video not found: #{id} : #{ex.message}")
       return error_template(404, ex)
@@ -128,10 +136,12 @@ module Invidious::Routes::Watch
     end
 
     if params.local
-      fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
-      adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+      fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
     end
 
+    # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
+    adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+
     video_streams = video.video_streams
     audio_streams = video.audio_streams
 

src/invidious/routing.cr

@@ -77,6 +77,7 @@ module Invidious::Routing
     post "/authorize_token", Routes::Account, :post_authorize_token
     get "/token_manager", Routes::Account, :token_manager
     post "/token_ajax", Routes::Account, :token_ajax
+    get "/generate_tokens", Routes::Account, :generate_tokens
     post "/subscription_ajax", Routes::Subscriptions, :toggle_subscription
     get "/subscription_manager", Routes::Subscriptions, :subscription_manager
   end

src/invidious/user/preferences.cr

@@ -57,6 +57,10 @@ struct Preferences
   property volume : Int32 = CONFIG.default_user_preferences.volume
   property save_player_pos : Bool = CONFIG.default_user_preferences.save_player_pos
 
+  @[YAML::Field(converter: Preferences::ProcessString)]
+  property po_token : String = ""
+  property visitor_data : String = ""
+
   module BoolToString
     def self.to_json(value : String, json : JSON::Builder)
       json.string value

src/invidious/videos.cr

@@ -294,7 +294,7 @@ struct Video
   predicate_bool upcoming, isUpcoming
 end
 
-def get_video(id, refresh = true, region = nil, force_refresh = false)
+def get_video(id, refresh = true, region = nil, force_refresh = false, po_token = "", visitor_data = "")
   if (video = Invidious::Database::Videos.select(id)) && !region
     # If record was last updated over 10 minutes ago, or video has since premiered,
     # refresh (expire param in response lasts for 6 hours)
@@ -304,7 +304,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
        force_refresh ||
        video.schema_version != Video::SCHEMA_VERSION # cache control
       begin
-        video = fetch_video(id, region)
+        video = fetch_video(id, region, po_token, visitor_data)
         Invidious::Database::Videos.insert(video)
       rescue ex
         Invidious::Database::Videos.delete(id)
@@ -312,7 +312,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
       end
     end
   else
-    video = fetch_video(id, region)
+    video = fetch_video(id, region, po_token, visitor_data)
     Invidious::Database::Videos.insert(video) if !region
   end
 
@@ -320,11 +320,11 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
 rescue DB::Error
   # Avoid common `DB::PoolRetryAttemptsExceeded` error and friends
   # Note: All DB errors inherit from `DB::Error`
-  return fetch_video(id, region)
+  return fetch_video(id, region, po_token, visitor_data)
 end
 
-def fetch_video(id, region)
-  info = extract_video_info(video_id: id)
+def fetch_video(id, region, po_token, visitor_data)
+  info = extract_video_info(video_id: id, user_po_token: po_token, user_visitor_data: visitor_data)
 
   if reason = info["reason"]?
     if reason == "Video unavailable"

src/invidious/videos/parser.cr

@@ -50,12 +50,17 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
   }
 end
 
-def extract_video_info(video_id : String)
+def extract_video_info(video_id : String, user_po_token, user_visitor_data)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
 
+  redis_po_token, redis_visitor_data = Tokens.get_tokens
+
+  po_token = (user_po_token if !user_po_token.empty?) || redis_po_token || CONFIG.po_token
+  visitor_data = (user_visitor_data if !user_visitor_data.empty?) || redis_visitor_data || CONFIG.visitor_data
+
   # Fetch data from the player endpoint
-  player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
+  player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config, po_token: po_token, visitor_data: visitor_data)
 
   playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
 
@@ -104,22 +109,22 @@ def extract_video_info(video_id : String)
 
   # Don't use Android client if po_token is passed because po_token doesn't
   # work for Android client.
-  if reason.nil? && CONFIG.po_token.nil?
+  if reason.nil? && po_token.nil?
     # Fetch the video streams using an Android client in order to get the
     # decrypted URLs and maybe fix throttling issues (#2194). See the
     # following issue for an explanation about decrypted URLs:
     # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
     client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
-    new_player_response = try_fetch_streaming_data(video_id, client_config)
+    new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data)
   end
 
   # Last hope
   # Only trigger if reason found and po_token or didn't work wth Android client.
   # TvHtml5ScreenEmbed now requires sig helper for it to work but po_token is not required
   # if the IP address is not blocked.
-  if CONFIG.po_token && reason || CONFIG.po_token.nil? && new_player_response.nil?
+  if po_token.nil? && reason || po_token.nil? && new_player_response.nil?
     client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
-    new_player_response = try_fetch_streaming_data(video_id, client_config)
+    new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data)
   end
 
   # Replace player response and reset reason
@@ -140,7 +145,7 @@ def extract_video_info(video_id : String)
   if streaming_data = player_response["streamingData"]?
     %w[formats adaptiveFormats].each do |key|
       streaming_data.as_h[key]?.try &.as_a.each do |format|
-        format.as_h["url"] = JSON::Any.new(convert_url(format))
+        format.as_h["url"] = JSON::Any.new(convert_url(format, po_token))
       end
     end
 
@@ -153,9 +158,9 @@ def extract_video_info(video_id : String)
   return params
 end
 
-def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
+def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig, po_token, visitor_data) : Hash(String, JSON::Any)?
   LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
-  response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config)
+  response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config, po_token: po_token, visitor_data: visitor_data)
 
   playability_status = response["playabilityStatus"]["status"]
   LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
@@ -455,7 +460,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   return params
 end
 
-private def convert_url(fmt)
+private def convert_url(fmt, po_token)
   if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
     sp = cfr["sp"]
     url = URI.parse(cfr["url"])
@@ -473,7 +478,9 @@ private def convert_url(fmt)
   n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
   params["n"] = n if n
 
-  if token = CONFIG.po_token
+  if !po_token.nil?
+    params["pot"] = po_token
+  elsif token = CONFIG.po_token
     params["pot"] = token
   end
 

src/invidious/views/user/preferences.ecr

@@ -126,6 +126,24 @@
                 <input name="save_player_pos" id="save_player_pos" type="checkbox" <% if preferences.save_player_pos %>checked<% end %>>
             </div>
 
+            <% if !CONFIG.ignore_user_tokens %>
+			<div class="pure-control-group">
+                <label for="po_token"><%= translate(locale, "preferences_po_token") %></label>
+                <input name="po_token" id="po_token" type="text" value="<%= preferences.po_token %>">
+            </div>
+
+			<div class="pure-control-group">
+                <label for="visitor_data"><%= translate(locale, "preferences_visitor_data") %></label>
+                <input name="visitor_data" id="visitor_data" type="text" value="<%= preferences.visitor_data %>">
+            </div>
+
+        <% if env.get?("user") %>
+			<div class="pure-control-group">
+				<a href="/generate_tokens?referer=<%= URI.encode_www_form(referer) %>"><%= translate(locale, "Generate po_token and visitor_data for your account") %></a>
+			</div>
+        <% end %>
+        <% end %>
+
             <legend><%= translate(locale, "preferences_category_visual") %></legend>
 
             <div class="pure-control-group">

src/invidious/views/user/tokens.ecr

@@ -0,0 +1,15 @@
+<% content_for "header" do %>
+<title><%= translate(locale, "Invidious token generator") %> - Invidious</title>
+<% end %>
+
+<div class="pure-g">
+    <div class="pure-u-1 pure-u-lg-1-5"></div>
+    <div class="pure-u-1 pure-u-lg-3-5">
+        <div class="h-box">
+		<p>po_token and visitor_data successfully generated!</p>
+		<p>po_token: <%= po_token %></p>
+		<p>visitor_data: <%= visitor_data %></p>
+        </div>
+    </div>
+    <div class="pure-u-1 pure-u-lg-1-5"></div>
+</div>

src/invidious/yt_backend/connection_pool.cr

@@ -1,10 +1,10 @@
 def add_yt_headers(request)
   request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
-  request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
+  request.headers["User-Agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
 
   request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
-  request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
-  request.headers["Accept-Language"] ||= "en-us,en;q=0.5"
+  request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
+  request.headers["Accept-Language"] ||= "en-US,en;q=0.9"
 
   # Preserve original cookies and add new YT consent cookie for EU servers
   request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}"

src/invidious/yt_backend/youtube_api.cr

@@ -3,6 +3,8 @@
 #
 
 module YoutubeAPI
+  @@visitor_data : String = ""
+
   extend self
 
   # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history
@@ -320,7 +322,9 @@ module YoutubeAPI
       client_context["client"]["platform"] = platform
     end
 
-    if CONFIG.visitor_data.is_a?(String)
+    if !@@visitor_data.empty?
+      client_context["client"]["visitorData"] = @@visitor_data
+    elsif CONFIG.visitor_data.is_a?(String)
       client_context["client"]["visitorData"] = CONFIG.visitor_data.as(String)
     end
 
@@ -455,8 +459,13 @@ module YoutubeAPI
     video_id : String,
     *, # Force the following parameters to be passed by name
     params : String,
-    client_config : ClientConfig | Nil = nil
+    client_config : ClientConfig | Nil = nil,
+    po_token : String | Nil,
+    visitor_data : String | Nil,
   )
+    if visitor_data
+      @@visitor_data = visitor_data
+    end
     # Playback context, separate because it can be different between clients
     playback_ctx = {
       "html5Preference" => "HTML5_PREF_WANTS",
@@ -482,7 +491,7 @@ module YoutubeAPI
         "contentPlaybackContext" => playback_ctx,
       },
       "serviceIntegrityDimensions" => {
-        "poToken" => CONFIG.po_token,
+        "poToken" => po_token || CONFIG.po_token,
       },
     }
 
@@ -596,7 +605,7 @@ module YoutubeAPI
   def _post_json(
     endpoint : String,
     data : Hash,
-    client_config : ClientConfig | Nil
+    client_config : ClientConfig | Nil,
   ) : Hash(String, JSON::Any)
     # Use the default client config if nil is passed
     client_config ||= DEFAULT_CLIENT_CONFIG
@@ -616,7 +625,9 @@ module YoutubeAPI
       headers["User-Agent"] = user_agent
     end
 
-    if CONFIG.visitor_data.is_a?(String)
+    if !@@visitor_data.empty?
+      headers["X-Goog-Visitor-Id"] = @@visitor_data
+    elsif CONFIG.visitor_data.is_a?(String)
       headers["X-Goog-Visitor-Id"] = CONFIG.visitor_data.as(String)
     end