Another attempt

Signed-off-by: Fijxu <fijxu@nadeko.net>

.forgejo/workflows/ci.yml

@@ -8,6 +8,8 @@ on:
   push:
     branches:
       - "master"
+      - "experimental"
+      - "experimental2"
 
 jobs:
   build:
@@ -32,8 +34,8 @@ jobs:
       with:
         images: git.nadeko.net/fijxu/invidious
         tags: |
-          type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
-          type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
+          type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-experimental-,enable=${{ github.ref == format('refs/heads/{0}', 'experimental') }}
+          type=raw,value=latest-experimental,enable=${{ github.ref == format('refs/heads/{0}', 'experimental') }}
 
     - uses: https://code.forgejo.org/docker/build-push-action@v5
       name: Build images

src/invidious/channels/channels.cr

@@ -251,8 +251,8 @@ def fetch_channel(ucid, pull_all_videos : Bool)
       LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
       if CONFIG.enable_user_notifications
         Invidious::Database::Users.add_notification(video)
-      else
-        Invidious::Database::Users.feed_needs_update(video)
+        #   else
+        #     Invidious::Database::Users.feed_needs_update(video)
       end
     else
       LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
@@ -287,8 +287,8 @@ def fetch_channel(ucid, pull_all_videos : Bool)
           if was_insert
             if CONFIG.enable_user_notifications
               Invidious::Database::Users.add_notification(video)
-            else
-              Invidious::Database::Users.feed_needs_update(video)
+              # else
+              #   Invidious::Database::Users.feed_needs_update(video)
             end
           end
         end

src/invidious/database/users.cr

@@ -154,15 +154,17 @@ module Invidious::Database::Users
   #  Update (misc)
   # -------------------
 
-  def feed_needs_update(video : ChannelVideo)
-    request = <<-SQL
-      UPDATE users
-      SET feed_needs_update = true
-      WHERE $1 = ANY(subscriptions)
-    SQL
+  # Feeds never need update. PubSubHubBub is the one that sends videos to
+  # invidious.
+  #   def feed_needs_update(video : ChannelVideo)
+  #     request = <<-SQL
+  #       UPDATE users
+  #       SET feed_needs_update = true
+  #       WHERE $1 = ANY(subscriptions)
+  #     SQL
 
-    PG_DB.exec(request, video.ucid)
-  end
+  #     PG_DB.exec(request, video.ucid)
+  #   end
 
   def update_preferences(user : User)
     request = <<-SQL

src/invidious/http_server/utils.cr

@@ -4,6 +4,17 @@ module Invidious::HttpServer
   module Utils
     extend self
 
+    @@proxy_alive : Bool = false
+
+    def check_external_proxy
+      begin
+        response = HTTP::Client.get("#{CONFIG.external_videoplayback_proxy}")
+        @@proxy_alive = response.status_code == 200
+      rescue
+        @@proxy_alive = false
+      end
+    end
+
     def proxy_video_url(raw_url : String, *, region : String? = nil, absolute : Bool = false)
       url = URI.parse(raw_url)
 
@@ -14,7 +25,11 @@ module Invidious::HttpServer
       url.query_params = params
 
       if absolute
-        return "#{HOST_URL}#{url.request_target}"
+        if @@proxy_alive
+          return "#{CONFIG.external_videoplayback_proxy}#{url.request_target}"
+        else
+          return "#{HOST_URL}#{url.request_target}"
+        end
       else
         return url.request_target
       end

src/invidious/jobs/check_external_proxy.cr

@@ -4,7 +4,7 @@ class Invidious::Jobs::CheckExternalProxy < Invidious::Jobs::BaseJob
 
   def begin
     loop do
-      Invidious::Routes::API::Manifest.check_external_proxy
+      HttpServer::Utils.check_external_proxy
       LOGGER.info("CheckExternalProxy: Done, sleeping for 1 minute")
       sleep 1.minutes
       Fiber.yield

src/invidious/routes/api/manifest.cr

@@ -1,15 +1,4 @@
 module Invidious::Routes::API::Manifest
-  @@proxy_alive : Bool = false
-
-  def self.check_external_proxy
-    begin
-      response = HTTP::Client.get("#{CONFIG.external_videoplayback_proxy}")
-      @@proxy_alive = response.status_code == 200
-    rescue
-      @@proxy_alive = false
-    end
-  end
-
   # /api/manifest/dash/id/:id
   def self.get_dash_video_id(env)
     env.response.headers.add("Access-Control-Allow-Origin", "*")
@@ -38,36 +27,21 @@ module Invidious::Routes::API::Manifest
         haltf env, status_code: response.status_code
       end
 
-      manifest = response.body
-
-      manifest = manifest.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
-        url = baseurl.lchop("<BaseURL>")
-        url = url.rchop("</BaseURL>")
-
-        if local
-          uri = URI.parse(url)
-          if @@proxy_alive
-            url = "#{CONFIG.external_videoplayback_proxy}#{uri.request_target}host/#{uri.host}/"
-          else
-            url = "#{HOST_URL}#{uri.request_target}host/#{uri.host}/"
-          end
-        end
-
+      # Proxy URLs for video playback on invidious.
+      # Other API clients can get the original URLs by omiting `local=true`.
+      manifest = response.body.gsub(/<BaseURL>[^<]+<\/BaseURL>/) do |baseurl|
+        url = baseurl.lchop("<BaseURL>").rchop("</BaseURL>")
+        url = HttpServer::Utils.proxy_video_url(url, absolute: true) if local
         "<BaseURL>#{url}</BaseURL>"
       end
 
       return manifest
     end
 
-    adaptive_fmts = video.adaptive_fmts
-
+    # Ditto, only proxify URLs if `local=true` is used
     if local
-      adaptive_fmts.each do |fmt|
-        if @@proxy_alive
-          fmt["url"] = JSON::Any.new("#{CONFIG.external_videoplayback_proxy}#{URI.parse(fmt["url"].as_s).request_target}")
-        else
-          fmt["url"] = JSON::Any.new("#{HOST_URL}#{URI.parse(fmt["url"].as_s).request_target}")
-        end
+      video.adaptive_fmts.each do |fmt|
+        fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s, absolute: true))
       end
     end
 
@@ -203,8 +177,9 @@ module Invidious::Routes::API::Manifest
     manifest = response.body
 
     if local
-      manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
-        path = URI.parse(match).path
+      manifest = manifest.gsub(/^https:\/\/[^\n"]*/m) do |match|
+        uri = URI.parse(match)
+        path = uri.path
 
         path = path.lchop("/videoplayback/")
         path = path.rchop("/")
@@ -233,7 +208,7 @@ module Invidious::Routes::API::Manifest
           raw_params["fvip"] = fvip["fvip"]
         end
 
-        raw_params["local"] = "true"
+        raw_params["host"] = uri.host.not_nil!
 
         "#{HOST_URL}/videoplayback?#{raw_params}"
       end

src/invidious/routes/embed.cr

@@ -157,10 +157,12 @@ module Invidious::Routes::Embed
     adaptive_fmts = video.adaptive_fmts
 
     if params.local
-      fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
-      adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+      fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
     end
 
+    # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
+    adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+
     video_streams = video.video_streams
     audio_streams = video.audio_streams
 

src/invidious/routes/feeds.cr

@@ -450,8 +450,8 @@ module Invidious::Routes::Feeds
         if was_insert
           if CONFIG.enable_user_notifications
             Invidious::Database::Users.add_notification(video)
-          else
-            Invidious::Database::Users.feed_needs_update(video)
+            #   else
+            #     Invidious::Database::Users.feed_needs_update(video)
           end
         end
       end

src/invidious/routes/video_playback.cr

@@ -42,9 +42,9 @@ module Invidious::Routes::VideoPlayback
       headers["Range"] = "bytes=#{range_for_head}"
     end
 
-		headers["Origin"] = "https://www.youtube.com"
-		headers["Referer"] = "https://www.youtube.com/"
-		headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64; rv:122.0) Gecko/20100101 Firefox/122.0"
+    headers["Origin"] = "https://www.youtube.com"
+    headers["Referer"] = "https://www.youtube.com/"
+    headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
 
     client = make_client(URI.parse(host), region, force_resolve = true)
     response = HTTP::Client::Response.new(500)

src/invidious/routes/watch.cr

@@ -128,10 +128,12 @@ module Invidious::Routes::Watch
     end
 
     if params.local
-      fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
-      adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+      fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
     end
 
+    # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
+    adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+
     video_streams = video.video_streams
     audio_streams = video.audio_streams
 

src/invidious/videos.cr

@@ -324,7 +324,7 @@ rescue DB::Error
 end
 
 def fetch_video(id, region)
-  info = extract_video_info(video_id: id)
+  info = extract_video_info(video_id: id, level: 0)
 
   if reason = info["reason"]?
     if reason == "Video unavailable"

src/invidious/videos/parser.cr

@@ -50,13 +50,22 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
   }
 end
 
-def extract_video_info(video_id : String)
+def extract_video_info(video_id : String, *, level = 0, client_type = YoutubeAPI::ClientType::WebMobileT2)
+  # Infinite recursion prevention
+  level += 1
+  if level >= 3
+    return {
+      "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
+      "reason"  => JSON::Any.new("All counter-measures exhausted"),
+    }
+  end
+
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
+  client_config.client_type = client_type
 
   # Fetch data from the player endpoint
   player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
-
   playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
 
   if playability_status != "OK"
@@ -65,10 +74,15 @@ def extract_video_info(video_id : String)
     reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
     reason ||= player_response.dig("playabilityStatus", "reason").as_s
 
-    # Stop here if video is not a scheduled livestream or
-    # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help
-    if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) ||
+    # Show the playability status in the reason message
+    reason = "#{reason} (#{playability_status})"
+
+    if (playability_status == "UNPLAYABLE" && reason.includes?("Get the YouTube app")) || reason.includes?("protect our community")
+      return extract_video_info(video_id: video_id, level: level, client_type: YoutubeAPI::ClientType::IOS)
+    elsif !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) ||
        playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails")
+      # Stop here if video is not a scheduled livestream or
+      # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help
       return {
         "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
         "reason"  => JSON::Any.new(reason),
@@ -92,7 +106,7 @@ def extract_video_info(video_id : String)
   end
 
   # Don't fetch the next endpoint if the video is unavailable.
-  if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
+  if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED", "UNPLAYABLE"}.any?(playability_status)
     next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
     player_response = player_response.merge(next_response)
   end
@@ -102,22 +116,12 @@ def extract_video_info(video_id : String)
 
   new_player_response = nil
 
-  # Don't use Android client if po_token is passed because po_token doesn't
-  # work for Android client.
-  if reason.nil? && CONFIG.po_token.nil?
-    # Fetch the video streams using an Android client in order to get the
-    # decrypted URLs and maybe fix throttling issues (#2194). See the
-    # following issue for an explanation about decrypted URLs:
-    # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
-    client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
+  if reason || DECRYPT_FUNCTION.nil? || CONFIG.po_token.nil?
+    client_config.client_type = YoutubeAPI::ClientType::IOS
     new_player_response = try_fetch_streaming_data(video_id, client_config)
   end
 
-  # Last hope
-  # Only trigger if reason found and po_token or didn't work wth Android client.
-  # TvHtml5ScreenEmbed now requires sig helper for it to work but po_token is not required
-  # if the IP address is not blocked.
-  if CONFIG.po_token && reason || CONFIG.po_token.nil? && new_player_response.nil?
+  if reason && !DECRYPT_FUNCTION.nil? && CONFIG.po_token.nil?
     client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
     new_player_response = try_fetch_streaming_data(video_id, client_config)
   end
@@ -470,11 +474,15 @@ private def convert_url(fmt)
     params = url.query_params
   end
 
-  n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
-  params["n"] = n if n
+  if old_n = params["n"]?
+    n = DECRYPT_FUNCTION.try &.decrypt_nsig(old_n)
+    params["n"] = n if n
+  end
 
   if token = CONFIG.po_token
-    params["pot"] = token
+    if {"WEB", "TVHTML5"}.any? { |x| params["c"].starts_with?(x) }
+      params["pot"] = token
+    end
   end
 
   url.query_params = params

src/invidious/views/components/player.ecr

@@ -4,9 +4,6 @@
     <% if params.autoplay %>autoplay<% end %>
     <% if params.video_loop %>loop<% end %>
     <% if params.controls %>controls<% end %>>
-    <% if (hlsvp = video.hls_manifest_url) && !CONFIG.disabled?("livestreams") %>
-        <source src="<%= URI.parse(hlsvp).request_target %><% if params.local %>?local=true<% end %>" type="application/x-mpegURL" label="livestream">
-    <% else %>
         <% if params.listen %>
             <% # default to 128k m4a stream
                best_m4a_stream_index = 0
@@ -57,6 +54,11 @@
             <% end %>
         <% end %>
 
+    <% if (hlsvp = video.hls_manifest_url) && !CONFIG.disabled?("livestreams") %>
+        <source src="<%= URI.parse(hlsvp).request_target %><% if params.local %>?local=true<% end %>" type="application/x-mpegURL" label="livestream">
+    <% end %>
+
+
         <% preferred_captions.each do |caption| %>
             <track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name %>" label="<%= caption.name %>">
         <% end %>
@@ -64,7 +66,6 @@
         <% captions.each do |caption| %>
             <track kind="captions" src="/api/v1/captions/<%= video.id %>?label=<%= caption.name %>" label="<%= caption.name %>">
         <% end %>
-    <% end %>
 </video>
 
 <script id="player_data" type="application/json">

src/invidious/yt_backend/connection_pool.cr

@@ -1,10 +1,10 @@
 def add_yt_headers(request)
   request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
-  request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
+  request.headers["User-Agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
 
   request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
-  request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
-  request.headers["Accept-Language"] ||= "en-us,en;q=0.5"
+  request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
+  request.headers["Accept-Language"] ||= "en-US,en;q=0.9"
 
   # Preserve original cookies and add new YT consent cookie for EU servers
   request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}"

src/invidious/yt_backend/youtube_api.cr

@@ -17,7 +17,7 @@ module YoutubeAPI
   # For Apple device names, see https://gist.github.com/adamawolf/3048717
   # For iOS versions, see https://en.wikipedia.org/wiki/IOS_version_history#Releases,
   # then go to the dedicated article of the major version you want.
-  private IOS_APP_VERSION = "19.32.8"
+  private IOS_APP_VERSION = "19.40.4"
   private IOS_USER_AGENT  = "com.google.ios.youtube/#{IOS_APP_VERSION} (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)"
   private IOS_VERSION     = "17.6.1.21G93" # Major.Minor.Patch.Build
 
@@ -28,6 +28,7 @@ module YoutubeAPI
     Web
     WebEmbeddedPlayer
     WebMobile
+    WebMobileT2
     WebScreenEmbed
 
     Android
@@ -48,7 +49,7 @@ module YoutubeAPI
     ClientType::Web => {
       name:       "WEB",
       name_proto: "1",
-      version:    "2.20240814.00.00",
+      version:    "2.20241010.01.00",
       screen:     "WATCH_FULL_SCREEN",
       os_name:    "Windows",
       os_version: WINDOWS_VERSION,
@@ -66,11 +67,19 @@ module YoutubeAPI
     ClientType::WebMobile => {
       name:       "MWEB",
       name_proto: "2",
-      version:    "2.20240813.02.00",
+      version:    "2.20241010.02.00",
       os_name:    "Android",
       os_version: ANDROID_VERSION,
       platform:   "MOBILE",
     },
+    ClientType::WebMobileT2 => {
+      name:       "MWEB_TIER_2",
+      name_proto: "27",
+      version:    "9.20241010",
+      os_name:    "iPhone",
+      os_version: IOS_VERSION,
+      platform:   "MOBILE",
+    },
     ClientType::WebScreenEmbed => {
       name:       "WEB",
       name_proto: "1",