Merge branch 'invidious-companion'

config/config.example.yml

@@ -54,6 +54,23 @@ db:
 ##
 #signature_server:
 
+##
+## Path to the Invidious companion.
+## An external program for loading the video streams from YouTube servers.
+##
+## When this setting is commented out, Invidious companion is not used.
+##
+## When this setting is configured and "external_port" is used then
+## you need to configure Invidious companion routes into your reverse proxy.
+## If "external_port" is not configured then Invidious will proxy the requests
+## to Invidious companion.
+##
+## Accepted values: "http(s)://<IP-HOSTNAME>:<Port>"
+## Default: <none>
+##
+# invidious_companion:
+# - http://127.0.0.1:8282
+
 
 #########################################
 #

src/invidious.cr

@@ -175,6 +175,12 @@ Invidious::Database.check_integrity(CONFIG)
   {% puts "\nDone checking player dependencies, now compiling Invidious...\n" %}
 {% end %}
 
+# invidious_companion and signature_server can't work together
+if CONFIG.signature_server && CONFIG.invidious_companion
+  puts "You can not run inv_sig_helper and invidious_companion at the same time."
+  exit(1)
+end
+
 # Misc
 
 DECRYPT_FUNCTION =

src/invidious/config.cr

@@ -177,6 +177,9 @@ class Config
   # poToken for passing bot attestation
   property po_token : String? = nil
 
+  # Invidious companion
+  property invidious_companion : Array(String)? = nil
+
   # Saved cookies in "name1=value1; name2=value2..." format
   @[YAML::Field(converter: Preferences::StringToCookies)]
   property cookies : HTTP::Cookies = HTTP::Cookies.new

src/invidious/routes/api/manifest.cr

@@ -20,6 +20,10 @@ module Invidious::Routes::API::Manifest
       haltf env, status_code: 403
     end
 
+    if local && CONFIG.invidious_companion
+      return env.redirect "#{video.invidious_companion["baseUrl"].as_s}#{env.request.path}?#{env.request.query}"
+    end
+
     if dashmpd = video.dash_manifest_url
       response = YT_POOL.client &.get(URI.parse(dashmpd).request_target)
 

src/invidious/routes/watch.cr

@@ -218,6 +218,13 @@ module Invidious::Routes::Watch
       captions: video.captions
     )
 
+    if (CONFIG.invidious_companion && (preferences.local || preferences.quality == "dash"))
+      env.response.headers["Content-Security-Policy"] =
+        env.response.headers["Content-Security-Policy"]
+          .gsub("media-src", "media-src " + video.invidious_companion["baseUrl"].as_s)
+          .gsub("connect-src", "connect-src " + video.invidious_companion["baseUrl"].as_s)
+    end
+
     templated "watch"
   end
 

src/invidious/videos.cr

@@ -192,6 +192,10 @@ struct Video
     }
   end
 
+  def invidious_companion : Hash(String, JSON::Any)
+    info["invidiousCompanion"].try &.as_h
+  end
+
   # Macros defining getters/setters for various types of data
 
   private macro getset_string(name)

src/invidious/videos/parser.cr

@@ -105,36 +105,44 @@ def extract_video_info(video_id : String, user_po_token, user_visitor_data)
   params = parse_video_info(video_id, player_response)
   params["reason"] = JSON::Any.new(reason) if reason
 
-  new_player_response = nil
+  if CONFIG.invidious_companion.nil?
+    new_player_response = nil
 
-  # Don't use Android client if po_token is passed because po_token doesn't
-  # work for Android client.
-  if reason.nil? && po_token.nil?
-    # Fetch the video streams using an Android client in order to get the
-    # decrypted URLs and maybe fix throttling issues (#2194). See the
-    # following issue for an explanation about decrypted URLs:
-    # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
-    client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
-    new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data)
-  end
+    # Second try in case WEB_CREATOR doesn't work with po_token.
+    # Only trigger if reason found and po_token configured.
+    if reason && CONFIG.po_token
+      client_config.client_type = YoutubeAPI::ClientType::WebEmbeddedPlayer
+      new_player_response = try_fetch_streaming_data(video_id, client_config)
+    end
 
-  # Last hope
-  # Only trigger if reason found and po_token or didn't work wth Android client.
-  # TvHtml5ScreenEmbed now requires sig helper for it to work but po_token is not required
-  # if the IP address is not blocked.
-  if po_token.nil? && reason || po_token.nil? && new_player_response.nil?
-    client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
-    new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data)
-  end
+    # Don't use Android client if po_token is passed because po_token doesn't
+    # work for Android client.
+    if reason.nil? && CONFIG.po_token.nil?
+      # Fetch the video streams using an Android client in order to get the
+      # decrypted URLs and maybe fix throttling issues (#2194). See the
+      # following issue for an explanation about decrypted URLs:
+      # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
+      client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
+      new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data)
+    end
 
-  # Replace player response and reset reason
-  if !new_player_response.nil?
-    # Preserve captions & storyboard data before replacement
-    new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]?
-    new_player_response["captions"] = player_response["captions"] if player_response["captions"]?
+    # Last hope
+    # Only trigger if reason found or didn't work wth Android client.
+    # TvHtml5ScreenEmbed now requires sig helper for it to work but doesn't work with po_token.
+    if reason && CONFIG.po_token.nil?
+      client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
+      new_player_response = try_fetch_streaming_data(video_id, client_config, po_token, visitor_data)
+    end
 
-    player_response = new_player_response
-    params.delete("reason")
+    # Replace player response and reset reason
+    if !new_player_response.nil?
+      # Preserve captions & storyboard data before replacement
+      new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]?
+      new_player_response["captions"] = player_response["captions"] if player_response["captions"]?
+
+      player_response = new_player_response
+      params.delete("reason")
+    end
   end
 
   {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
@@ -152,6 +160,10 @@ def extract_video_info(video_id : String, user_po_token, user_visitor_data)
     params["streamingData"] = streaming_data
   end
 
+  if CONFIG.invidious_companion
+    params["invidiousCompanion"] = player_response["invidiousCompanion"]
+  end
+
   # Data structure version, for cache control
   params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64)
 
@@ -450,11 +462,12 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     # Music section
     "music" => JSON.parse(music_list.to_json),
     # Author infos
-    "author"          => JSON::Any.new(author || ""),
-    "ucid"            => JSON::Any.new(ucid || ""),
-    "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""),
-    "authorVerified"  => JSON::Any.new(author_verified || false),
-    "subCountText"    => JSON::Any.new(subs_text || "-"),
+    "author"             => JSON::Any.new(author || ""),
+    "ucid"               => JSON::Any.new(ucid || ""),
+    "authorThumbnail"    => JSON::Any.new(author_thumbnail.try &.as_s || ""),
+    "authorVerified"     => JSON::Any.new(author_verified || false),
+    "subCountText"       => JSON::Any.new(subs_text || "-"),
+    "invidiousCompanion" => JSON::Any.new(subs_text),
   }
 
   return params

src/invidious/views/components/player.ecr

@@ -22,6 +22,7 @@
                audio_streams.each_with_index do |fmt, i|
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
+                src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local)
 
                 bitrate = fmt["bitrate"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)
@@ -34,8 +35,11 @@
                 <% end %>
             <% end %>
         <% else %>
-            <% if params.quality == "dash" %>
-                <source src="/api/manifest/dash/id/<%= video.id %>?local=true&unique_res=1" type='application/dash+xml' label="dash">
+            <% if params.quality == "dash"
+               src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1"
+               src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion)
+            %>
+                <source src="<%= src_url %>" type='application/dash+xml' label="dash">
             <% end %>
 
             <%
@@ -44,6 +48,7 @@
             fmt_stream.each_with_index do |fmt, i|
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
+                src_url = video.invidious_companion["baseUrl"].as_s + src_url if (CONFIG.invidious_companion && params.local)
 
                 quality = fmt["quality"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)

src/invidious/yt_backend/youtube_api.cr

@@ -615,12 +615,19 @@ module YoutubeAPI
 
     headers = HTTP::Headers{
       "Content-Type"              => "application/json; charset=UTF-8",
-      "Accept-Encoding"           => "gzip, deflate",
       "x-goog-api-format-version" => "2",
       "x-youtube-client-name"     => client_config.name_proto,
       "x-youtube-client-version"  => client_config.version,
     }
 
+    if CONFIG.invidious_companion && endpoint == "/youtubei/v1/player"
+      headers["Authorization"] = "Bearer " + CONFIG.hmac_key
+    end
+
+    if !CONFIG.invidious_companion
+      headers["Accept-Encoding"] = "gzip, deflate"
+    end
+
     if user_agent = client_config.user_agent
       headers["User-Agent"] = user_agent
     end
@@ -636,11 +643,32 @@ module YoutubeAPI
     LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}")
     LOGGER.trace("YoutubeAPI: POST data: #{data}")
 
+    invidious_companion_urls = CONFIG.invidious_companion
+
     # Send the POST request
-    body = YT_POOL.client() do |client|
-      client.post(url, headers: headers, body: data.to_json) do |response|
-        self._decompress(response.body_io, response.headers["Content-Encoding"]?)
+    if invidious_companion_urls && endpoint == "/youtubei/v1/player"
+      puts "invidious companion section"
+      puts invidious_companion_urls[Random.rand(invidious_companion_urls.size)]
+      begin
+        invidious_companion_response = make_client(URI.parse(invidious_companion_urls[Random.rand(invidious_companion_urls.size)]),
+          &.post(endpoint, headers: headers, body: data.to_json))
+        body = invidious_companion_response.body
+        if (invidious_companion_response.status_code != 200)
+          raise Exception.new("status code: " + invidious_companion_response.status_code.to_s + " and body: " + body)
+        end
+      rescue ex
+        raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found"))
       end
+    else
+      body = YT_POOL.client() do |client|
+        client.post(url, headers: headers, body: data.to_json) do |response|
+          self._decompress(response.body_io, response.headers["Content-Encoding"]?)
+        end
+      end
+    end
+
+    if body.nil? && CONFIG.invidious_companion
+      raise InfoException.new("Error while communicating with Invidious companion: no response data.")
     end
 
     # Convert result to Hash