add support for external token generator

Dockerfile

@@ -1,4 +1,4 @@
-FROM denoland/deno:debian-2.1.4 AS builder
+FROM i.sanxian.tech/denoland/deno:debian-2.1.3 AS builder
 
 ARG TINI_VERSION=0.19.0
 

README.md

@@ -6,12 +6,8 @@ Companion for Invidious which handle all the video stream retrieval from YouTube
 
 - [deno](https://docs.deno.com/runtime/)  
 
-## Documentation
-- Installation guide: https://docs.invidious.io/companion-installation/
-- Extra documentation for Invidious companion: https://github.com/iv-org/invidious-companion/wiki
-
-## Run Locally (development)
+## Run Locally
 
 ```
 SERVER_SECRET_KEY=CHANGEME deno task dev
-```
+```

config/default.toml

@@ -1,11 +1,9 @@
 [server]
 port = 8282
 host = "127.0.0.1"
-# secret key needs to be 16 characters long or more
-secret_key = "CHANGE_ME"
+secret_key = "myBeautifulKey"
 base_url = "http://localhost:8282"
 verify_requests = false
-# max_dash_resolution = 1080
 
 [cache]
 enabled = true

deno.json

@@ -7,7 +7,6 @@
     "hono": "jsr:@hono/hono@^4.6.5",
     "hono/logger": "jsr:@hono/hono@^4.6.5/logger",
     "hono/bearer-auth": "jsr:@hono/hono@^4.6.5/bearer-auth",
-    "prom-client": "npm:prom-client@^15.1.3",
     "youtubei.js": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v12.2.0-deno/deno.ts",
     "youtubei.js/Utils": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v12.2.0-deno/deno/src/utils/Utils.ts",
     "youtubei.js/NavigationEndpoint": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v12.2.0-deno/deno/src/parser/classes/NavigationEndpoint.ts",

src/lib/helpers/youtubePlayerHandling.ts

@@ -2,7 +2,6 @@ import { ApiResponse, Innertube, YT } from "youtubei.js";
 import { generateRandomString } from "youtubei.js/Utils";
 import { compress, decompress } from "https://deno.land/x/brotli@0.1.7/mod.ts";
 import { Store } from "@willsoto/node-konfig-core";
-import { cachedEntries } from "../../routes/index.ts";
 let youtubePlayerReqLocation = "youtubePlayerReq";
 if (Deno.env.get("YT_PLAYER_REQ_LOCATION")) {
     if (Deno.env.has("DENO_COMPILED")) {
@@ -16,15 +15,7 @@ if (Deno.env.get("YT_PLAYER_REQ_LOCATION")) {
 }
 const { youtubePlayerReq } = await import(youtubePlayerReqLocation);
 
-const videoCachePath = Deno.env.get("VIDEO_CACHE_PATH") as string || "/var/tmp/youtubei.js/video_cache"
-
-let kv : Deno.Kv
-if ((Deno.env.get("VIDEO_CACHE_ON_DISK")?.toLowerCase() ?? false) == "true") {
-    console.log("[INFO] Storing video cache on disk")
-    kv = await Deno.openKv(videoCachePath);
-} else {
-    kv = await Deno.openKv();
-}
+const kv = await Deno.openKv();
 
 export const youtubePlayerParsing = async (
     innertubeClient: Innertube,
@@ -81,11 +72,6 @@ export const youtubePlayerParsing = async (
                         delete videoData.streamingData.formats[index]
                             .signatureCipher;
                     }
-                    if (videoData.streamingData.formats[index].url.includes("alr=yes")) {
-                        videoData.streamingData.formats[index].url.replace("alr=yes", "alr=no");
-                    } else {
-                        videoData.streamingData.formats[index].url += "&alr=no";
-                    }
                 }
                 for (
                     const [index, adaptive_format] of streamingData
@@ -105,11 +91,6 @@ export const youtubePlayerParsing = async (
                         delete videoData.streamingData.adaptiveFormats[index]
                             .signatureCipher;
                     }
-                    if (videoData.streamingData.adaptiveFormats[index].url.includes("alr=yes")) {
-                        videoData.streamingData.adaptiveFormats[index].url.replace("alr=yes", "alr=no");
-                    } else {
-                        videoData.streamingData.adaptiveFormats[index].url += "&alr=no";
-                    }
                 }
             }
         }
@@ -153,7 +134,6 @@ export const youtubePlayerParsing = async (
                         expireIn: 1000 * 60 * 60,
                     },
                 );
-                cachedEntries.inc()
             })();
         }
 

src/lib/jobs/potoken.ts

@@ -3,7 +3,6 @@ import type { BgConfig } from "bgutils";
 import { JSDOM } from "jsdom";
 import { Innertube, UniversalCache } from "youtubei.js";
 import { Store } from "@willsoto/node-konfig-core";
-import { poTokenFail, externalTokenGeneratorFail } from "../../routes/index.ts";
 let getFetchClientLocation = "getFetchClient";
 if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
     if (Deno.env.has("DENO_COMPILED")) {
@@ -26,38 +25,22 @@ export const poTokenGenerate = async (
     const externalTokenGenerator = konfigStore.get(
         "server.external_token_generator",
     ) as string;
-    const externalTokenGeneratorKey = konfigStore.get(
-        "server.external_token_generator_key",
-    ) as string;
     const requestKey = "O43z0dpjhgX20SCx4KAo";
 
     if (externalTokenGenerator != "" && externalTokenGenerator != undefined) {
         console.log(
-            "poTokenGenerate: using external token generator at " +
-                externalTokenGenerator,
+            "using external token generator at " + externalTokenGenerator,
         );
         try {
-            let response: Response;
-            if (
-                externalTokenGeneratorKey != "" &&
-                externalTokenGeneratorKey != undefined
-            ) {
-                response = await fetch(
-                    `${externalTokenGenerator}/generate`, {
-                        headers: {
-                            'Authorization': `Bearer ${externalTokenGeneratorKey}`
-                        }
-                    }
-                );
-                if (response.status == 401) {
-                    throw new Error(`Key '${externalTokenGeneratorKey}' is invalid!`)
-                }
-            } else {
-                response = await fetch(
-                    `${externalTokenGenerator}/generate`,
-                );
-            }
+            const response = await fetch(
+                `${externalTokenGenerator}/generate`,
+            );
             const data = await response.json();
+            // innertubeClient.session.po_token = data.potoken;
+            // innertubeClient.session.context.client.visitorData =
+            //     data.visitorData;
+            console.log(data.potoken);
+            console.log(data.visitorData);
             return (await Innertube.create({
                 po_token: data.potoken,
                 visitor_data: data.visitorData,
@@ -66,14 +49,30 @@ export const poTokenGenerate = async (
                 generate_session_locally: true,
             }));
         } catch (e) {
-            console.error(
-                "poTokenGenerate: error fetch token from the external token generator: " +
-                    e,
-            );
-            console.log("poTokenGenerate: Using built-in token generator");
-            externalTokenGeneratorFail.inc()
+            console.error("error fetch token from the external token generator: " + e);
         }
     }
+    // setInterval(() => {
+    //     fetch(
+    //         `${externalTokenGenerator}/generate`,
+    //     ).then((response) => response.json()).then((data) => {
+    //         innertubeClient.session.po_token = data.potoken;
+    //         innertubeClient.session.context.client.visitorData =
+    //             data.visitorData;
+    //         console.log("po_token: "+data.potoken);
+    //         console.log("visitor_data"+data.visitorData);
+    //         return (await Innertube.create({
+    //             po_token: poTokenResult.poToken,
+    //             visitor_data: visitorData,
+    //             fetch: getFetchClient(konfigStore),
+    //             cache: innertubeClientCache,
+    //             generate_session_locally: true,
+    //         }));
+    //     }).catch(e => {
+    //         console.error("error getting tokens from external token generator: " + e)
+    //         console.log("tokens will not change")
+    //     });
+    // }, 5000);
 
     if (innertubeClient.session.po_token) {
         innertubeClient = await Innertube.create({ retrieve_player: false });
@@ -102,7 +101,6 @@ export const poTokenGenerate = async (
     const bgChallenge = await BG.Challenge.create(bgConfig);
 
     if (!bgChallenge) {
-        poTokenFail.inc()
         throw new Error("Could not get challenge");
     }
 
@@ -111,10 +109,7 @@ export const poTokenGenerate = async (
 
     if (interpreterJavascript) {
         new Function(interpreterJavascript)();
-    } else { 
-        poTokenFail.inc()
-        throw new Error("Could not load VM");
-    }
+    } else throw new Error("Could not load VM");
 
     const poTokenResult = await BG.PoToken.generate({
         program: bgChallenge.program,
@@ -124,6 +119,8 @@ export const poTokenGenerate = async (
 
     await BG.PoToken.generatePlaceholder(visitorData);
 
+    console.log("realpotoken: "+poTokenResult.poToken)
+    console.log("realvisitordata: "+visitorData)
     return (await Innertube.create({
         po_token: poTokenResult.poToken,
         visitor_data: visitorData,

src/main.ts

@@ -67,12 +67,17 @@ if (!innertubeClientOauthEnabled) {
     }
     setInterval(
         async () => {
+            const currentDateTime = new Date();
+            console.log("regenrate token called: " + currentDateTime)
             if (innertubeClientJobPoTokenEnabled) {
                 innertubeClient = await poTokenGenerate(
                     innertubeClient,
                     konfigStore,
                     innertubeClientCache,
                 );
+                const currentDateTimexd = new Date();
+            console.log("regenerate token finished: " + currentDateTimexd)
+            console.log("po_token: " + innertubeClient.session.po_token)
             } else {
                 innertubeClient = await Innertube.create({
                     cache: innertubeClientCache,
@@ -80,9 +85,7 @@ if (!innertubeClientOauthEnabled) {
                 });
             }
         },
-        konfigStore.get("jobs.youtube_session.frequency_seconds") as number *
-                1000 || 50000,
-    );
+        konfigStore.get("jobs.youtube_session.frequency_seconds") as number * 1000 || 50000);
 } else if (innertubeClientOauthEnabled) {
     // Fired when waiting for the user to authorize the sign in attempt.
     innertubeClient.session.on("auth-pending", (data) => {
@@ -114,11 +117,11 @@ app.use("*", async (c, next) => {
 });
 
 routes(app, konfigStore);
-
 const https = Deno.env.get("HTTPS");
 const port = konfigStore.get("server.port") as number;
 const host = konfigStore.get("server.host") as string;
 
+
 if (https == "TRUE" || https == "true") {
     const cert = Deno.readTextFileSync("/data/cert.pem");
     const key = Deno.readTextFileSync("/data/key.key");

src/routes/health.ts

@@ -1,12 +0,0 @@
-import { Hono } from "hono";
-
-const health = new Hono();
-
-health.get("/", () => {
-  return new Response("OK", {
-	status: 200,
-    headers: { "Content-Type": "text/plain" },
-  });
-});
-
-export default health;

src/routes/index.ts

@@ -2,59 +2,11 @@ import { Hono } from "hono";
 import { logger } from "hono/logger";
 import { Store } from "@willsoto/node-konfig-core";
 import { bearerAuth } from "hono/bearer-auth";
-import { Registry, Counter } from "prom-client"
 
 import youtubeApiPlayer from "./youtube_api_routes/player.ts";
 import invidiousRouteLatestVersion from "./invidious_routes/latestVersion.ts";
 import invidiousRouteDashManifest from "./invidious_routes/dashManifest.ts";
 import videoPlaybackProxy from "./videoPlaybackProxy.ts";
-import metrics from "./metrics.ts";
-
-const METRICS_PREFIX = "invidious_companion_";
-export const register = new Registry();
-
-export const externalTokenGeneratorFail = new Counter({
-	name: `${METRICS_PREFIX}externaltokengenerator_fail`,
-	help: 'TODO',
-	registers: [register]
-  });
-
-export const cachedEntries = new Counter({
-	name: `${METRICS_PREFIX}cached_entries`,
-	help: 'TODO',
-	registers: [register]
-  });
-
-export const subreasonProtectCommunity = new Counter({
-	name: `${METRICS_PREFIX}subreason_protect_community`,
-	help: 'TODO',
-	registers: [register]
-  });
-
-export const poTokenFail = new Counter({
-	name: `${METRICS_PREFIX}potoken_fail`,
-	help: 'TODO',
-	registers: [register]
-  });
-
-export const reasonBot = new Counter({
-	name: `${METRICS_PREFIX}reason_bot`,
-	help: 'TODO',
-	registers: [register]
-  });
-
-export const videoUnavailable = new Counter({
-	name: `${METRICS_PREFIX}video_unavailable`,
-	help: 'TODO',
-	registers: [register]
-  });
-
-export const videoRestricted = new Counter({
-	name: `${METRICS_PREFIX}video_restricted`,
-	help: 'TODO',
-	registers: [register]
-  });
-import health from "./health.ts";
 
 export const routes = (app: Hono, konfigStore: Store<Record<string, unknown>>) => {
   app.use("*", logger());
@@ -70,6 +22,4 @@ export const routes = (app: Hono, konfigStore: Store<Record<string, unknown>>) =
   app.route("/latest_version", invidiousRouteLatestVersion);
   app.route("/api/manifest/dash/id", invidiousRouteDashManifest);
   app.route("/videoplayback", videoPlaybackProxy);
-  app.route("/metrics", metrics)
-  app.route("/healthz", health);
 };

src/routes/invidious_routes/dashManifest.ts

@@ -24,8 +24,6 @@ dashManifest.get("/:videoId", async (c) => {
         Record<string, unknown>
     >;
 
-    const maxDashResolution = Deno.env.get("SERVER_MAX_DASH_RESOLUTION") || konfigStore.get("server.max_dash_resolution") as number;
-
     if (konfigStore.get("server.verify_requests") && check == undefined) {
         throw new HTTPException(400, {
             res: new Response("No check ID."),
@@ -69,12 +67,7 @@ dashManifest.get("/:videoId", async (c) => {
                         ).includes("av01")
                     ) {
                         if (i.mime_type.includes("av01")) {
-                            // @ts-ignore 'i.height' is possibly 'undefined'.
-                            if (i.height > maxDashResolution) {
-                                return false;
-                            } else {
-                                return true;
-                            }
+                            return true;
                         } else {
                             return false;
                         }

src/routes/metrics.ts

@@ -1,12 +0,0 @@
-import { Hono } from "hono";
-import { register } from "./index.ts";
-
-const metrics = new Hono();
-  
-metrics.get("/", async () => {
-  return new Response(await register.metrics(), {
-    headers: { "Content-Type": "text/plain" },
-  });
-});
-
-export default metrics;

src/routes/videoPlaybackProxy.ts

@@ -52,6 +52,10 @@ videoPlaybackProxy.get("/", async (c) => {
     // deno-lint-ignore prefer-const
     let queryParams = new URLSearchParams(urlReq.search);
     queryParams.delete("host");
+    // alr parameter is only for WEB/HTML5 clients
+    if (client.includes("WEB")) {
+        queryParams.append("alr", "yes");
+    }
     if (rangeHeader) {
         queryParams.append(
             "range",

src/routes/youtube_api_routes/player.ts

@@ -3,33 +3,9 @@ import { youtubePlayerParsing } from "../../lib/helpers/youtubePlayerHandling.ts
 import { HonoVariables } from "../../lib/types/HonoVariables.ts";
 import { Innertube } from "youtubei.js";
 import { Store } from "@willsoto/node-konfig-core";
-import { reasonBot, subreasonProtectCommunity, videoRestricted, videoUnavailable } from "../index.ts";
 
 const player = new Hono<{ Variables: HonoVariables }>();
 
-const errors = [
-  {
-    // @ts-ignore: Property 'playabilityStatus' does not exist on type 'object'
-    check: (yt: object) => yt.playabilityStatus?.reason?.includes("Sign in to confirm you’re not a bot"),
-    action: () => reasonBot.inc(),
-  },
-  {
-    // @ts-ignore: Property 'playabilityStatus' does not exist on type 'object'
-    check: (yt: object) => yt.playabilityStatus?.errorScreen?.playerErrorMessageRenderer?.subreason?.runs?.[0]?.text?.includes("This helps protect our community"),
-    action: () => subreasonProtectCommunity.inc(),
-  },
-  {
-    // @ts-ignore: Property 'playabilityStatus' does not exist on type 'object'
-    check: (yt: object) => yt.playabilityStatus?.errorScreen?.playerErrorMessageRenderer?.subreason?.runs?.[0]?.text?.includes("Video unavailable"),
-    action: () => videoUnavailable.inc(),
-  },
-  {
-    // @ts-ignore: Property 'playabilityStatus' does not exist on type 'object'
-    check: (yt: object) => yt.playabilityStatus?.reason?.includes("This video is restricted"),
-    action: () => videoRestricted.inc(),
-  },
-];
-
 player.post("/player", async (c) => {
   const jsonReq = await c.req.json();
   const innertubeClient = await c.get("innertubeClient") as Innertube;
@@ -38,13 +14,9 @@ player.post("/player", async (c) => {
     Record<string, unknown>
   >;
   if (jsonReq.videoId) {
-    const yt = await youtubePlayerParsing(innertubeClient, jsonReq.videoId, konfigStore)
-    errors.forEach((error) => {
-      if (error.check(yt)) {
-        error.action()
-      }
-    })
-    return c.json(yt);
+    return c.json(
+      await youtubePlayerParsing(innertubeClient, jsonReq.videoId, konfigStore)
+    );
   }
 });