feat(proxy): ignore certificate errors for proxy

add environment variable for the resolution limit

Dockerfile

@@ -1,4 +1,4 @@
-FROM i.sanxian.tech/denoland/deno:debian-2.1.3 AS builder
+FROM denoland/deno:debian-2.1.4 AS builder
 
 ARG TINI_VERSION=0.19.0
 

README.md

@@ -6,8 +6,12 @@ Companion for Invidious which handle all the video stream retrieval from YouTube
 
 - [deno](https://docs.deno.com/runtime/)  
 
-## Run Locally
+## Documentation
+- Installation guide: https://docs.invidious.io/companion-installation/
+- Extra documentation for Invidious companion: https://github.com/iv-org/invidious-companion/wiki
+
+## Run Locally (development)
 
 ```
 SERVER_SECRET_KEY=CHANGEME deno task dev
-```
+```

config/default.toml

@@ -1,9 +1,11 @@
 [server]
 port = 8282
 host = "127.0.0.1"
-secret_key = "myBeautifulKey"
+# secret key needs to be 16 characters long or more
+secret_key = "CHANGE_ME"
 base_url = "http://localhost:8282"
 verify_requests = false
+# max_dash_resolution = 1080
 
 [cache]
 enabled = true

deno.json

@@ -1,12 +1,13 @@
 {
   "tasks": {
     "dev": "deno run --allow-import=github.com:443,jsr.io:443,raw.githubusercontent.com:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname --allow-read --allow-write=/var/tmp/youtubei.js --watch src/main.ts",
-    "compile": "deno compile --include ./src/lib/helpers/youtubePlayerReq.ts --include ./src/lib/helpers/getFetchClient.ts --output invidious_companion --allow-import=github.com:443,jsr.io:443,raw.githubusercontent.com:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-read --allow-sys=hostname --allow-write=/var/tmp/youtubei.js src/main.ts"
+    "compile": "deno compile --include ./src/lib/helpers/youtubePlayerReq.ts --include ./src/lib/helpers/getFetchClient.ts --output invidious_companion --allow-import=github.com:443,jsr.io:443,raw.githubusercontent.com:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-read --allow-sys=hostname --allow-write=/var/tmp/youtubei.js --unsafely-ignore-certificate-errors src/main.ts"
   },
   "imports": {
     "hono": "jsr:@hono/hono@^4.6.5",
     "hono/logger": "jsr:@hono/hono@^4.6.5/logger",
     "hono/bearer-auth": "jsr:@hono/hono@^4.6.5/bearer-auth",
+    "prom-client": "npm:prom-client@^15.1.3",
     "youtubei.js": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v12.2.0-deno/deno.ts",
     "youtubei.js/Utils": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v12.2.0-deno/deno/src/utils/Utils.ts",
     "youtubei.js/NavigationEndpoint": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v12.2.0-deno/deno/src/parser/classes/NavigationEndpoint.ts",

src/lib/helpers/youtubePlayerHandling.ts

@@ -2,6 +2,7 @@ import { ApiResponse, Innertube, YT } from "youtubei.js";
 import { generateRandomString } from "youtubei.js/Utils";
 import { compress, decompress } from "https://deno.land/x/brotli@0.1.7/mod.ts";
 import { Store } from "@willsoto/node-konfig-core";
+import { failedRequests, successfulRequests } from "../../routes/index.ts";
 let youtubePlayerReqLocation = "youtubePlayerReq";
 if (Deno.env.get("YT_PLAYER_REQ_LOCATION")) {
     if (Deno.env.has("DENO_COMPILED")) {
@@ -15,7 +16,15 @@ if (Deno.env.get("YT_PLAYER_REQ_LOCATION")) {
 }
 const { youtubePlayerReq } = await import(youtubePlayerReqLocation);
 
-const kv = await Deno.openKv();
+const videoCachePath = Deno.env.get("VIDEO_CACHE_PATH") as string || "/var/tmp/youtubei.js/video_cache"
+
+export let kv : Deno.Kv
+if ((Deno.env.get("VIDEO_CACHE_ON_DISK")?.toLowerCase() ?? false) == "true") {
+    console.log("[INFO] Storing video cache on disk")
+    kv = await Deno.openKv(videoCachePath);
+} else {
+    kv = await Deno.openKv();
+}
 
 export const youtubePlayerParsing = async (
     innertubeClient: Innertube,
@@ -72,6 +81,11 @@ export const youtubePlayerParsing = async (
                         delete videoData.streamingData.formats[index]
                             .signatureCipher;
                     }
+                    if (videoData.streamingData.formats[index].url.includes("alr=yes")) {
+                        videoData.streamingData.formats[index].url.replace("alr=yes", "alr=no");
+                    } else {
+                        videoData.streamingData.formats[index].url += "&alr=no";
+                    }
                 }
                 for (
                     const [index, adaptive_format] of streamingData
@@ -91,6 +105,11 @@ export const youtubePlayerParsing = async (
                         delete videoData.streamingData.adaptiveFormats[index]
                             .signatureCipher;
                     }
+                    if (videoData.streamingData.adaptiveFormats[index].url.includes("alr=yes")) {
+                        videoData.streamingData.adaptiveFormats[index].url.replace("alr=yes", "alr=no");
+                    } else {
+                        videoData.streamingData.adaptiveFormats[index].url += "&alr=no";
+                    }
                 }
             }
         }
@@ -119,22 +138,27 @@ export const youtubePlayerParsing = async (
             },
         }))(videoData);
 
-        if (
+        if (videoData.playabilityStatus?.status == "OK") {
-            cacheEnabled == true && videoData.playabilityStatus?.status == "OK"
+            successfulRequests.inc()
-        ) {
+            if (
-            (async () => {
+                cacheEnabled == true
-                await kv.set(
+            ) {
-                    ["video_cache", videoId],
+                (async () => {
-                    compress(
+                    await kv.set(
-                        new TextEncoder().encode(
+                        ["video_cache", videoId],
-                            JSON.stringify(videoOnlyNecessaryInfo),
+                        compress(
+                            new TextEncoder().encode(
+                                JSON.stringify(videoOnlyNecessaryInfo),
+                            ),
                         ),
-                    ),
+                        {
-                    {
+                            expireIn: 1000 * 60 * 60,
-                        expireIn: 1000 * 60 * 60,
+                        },
-                    },
+                    );
-                );
+                })();
-            })();
+            }
+        } else {
+            failedRequests.inc()
         }
 
         return videoOnlyNecessaryInfo;

src/lib/jobs/potoken.ts

@@ -3,6 +3,7 @@ import type { BgConfig } from "bgutils";
 import { JSDOM } from "jsdom";
 import { Innertube, UniversalCache } from "youtubei.js";
 import { Store } from "@willsoto/node-konfig-core";
+import { poTokenFail, externalTokenGeneratorFail } from "../../routes/index.ts";
 let getFetchClientLocation = "getFetchClient";
 if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
     if (Deno.env.has("DENO_COMPILED")) {
@@ -22,8 +23,58 @@ export const poTokenGenerate = async (
     konfigStore: Store<Record<string, unknown>>,
     innertubeClientCache: UniversalCache,
 ): Promise<Innertube> => {
+    const externalTokenGenerator = konfigStore.get(
+        "server.external_token_generator",
+    ) as string;
+    const externalTokenGeneratorKey = konfigStore.get(
+        "server.external_token_generator_key",
+    ) as string;
     const requestKey = "O43z0dpjhgX20SCx4KAo";
 
+    if (externalTokenGenerator != "" && externalTokenGenerator != undefined) {
+        console.log(
+            "poTokenGenerate: using external token generator at " +
+                externalTokenGenerator,
+        );
+        try {
+            let response: Response;
+            if (
+                externalTokenGeneratorKey != "" &&
+                externalTokenGeneratorKey != undefined
+            ) {
+                response = await fetch(
+                    `${externalTokenGenerator}/generate`, {
+                        headers: {
+                            'Authorization': `Bearer ${externalTokenGeneratorKey}`
+                        }
+                    }
+                );
+                if (response.status == 401) {
+                    throw new Error(`Key '${externalTokenGeneratorKey}' is invalid!`)
+                }
+            } else {
+                response = await fetch(
+                    `${externalTokenGenerator}/generate`,
+                );
+            }
+            const data = await response.json();
+            return (await Innertube.create({
+                po_token: data.potoken,
+                visitor_data: data.visitorData,
+                fetch: getFetchClient(konfigStore),
+                cache: innertubeClientCache,
+                generate_session_locally: true,
+            }));
+        } catch (e) {
+            console.error(
+                "poTokenGenerate: error fetch token from the external token generator: " +
+                    e,
+            );
+            console.log("poTokenGenerate: Using built-in token generator");
+            externalTokenGeneratorFail.inc()
+        }
+    }
+
     if (innertubeClient.session.po_token) {
         innertubeClient = await Innertube.create({ retrieve_player: false });
     }
@@ -51,6 +102,7 @@ export const poTokenGenerate = async (
     const bgChallenge = await BG.Challenge.create(bgConfig);
 
     if (!bgChallenge) {
+        poTokenFail.inc()
         throw new Error("Could not get challenge");
     }
 
@@ -59,7 +111,10 @@ export const poTokenGenerate = async (
 
     if (interpreterJavascript) {
         new Function(interpreterJavascript)();
-    } else throw new Error("Could not load VM");
+    } else { 
+        poTokenFail.inc()
+        throw new Error("Could not load VM");
+    }
 
     const poTokenResult = await BG.PoToken.generate({
         program: bgChallenge.program,

src/main.ts

@@ -67,17 +67,12 @@ if (!innertubeClientOauthEnabled) {
     }
     setInterval(
         async () => {
-            const currentDateTime = new Date();
-            console.log("regenrate token called: " + currentDateTime)
             if (innertubeClientJobPoTokenEnabled) {
                 innertubeClient = await poTokenGenerate(
                     innertubeClient,
                     konfigStore,
                     innertubeClientCache,
                 );
-                const currentDateTimexd = new Date();
-            console.log("regenerate token finished: " + currentDateTimexd)
-            console.log("po_token: " + innertubeClient.session.po_token)
             } else {
                 innertubeClient = await Innertube.create({
                     cache: innertubeClientCache,
@@ -85,7 +80,9 @@ if (!innertubeClientOauthEnabled) {
                 });
             }
         },
-        konfigStore.get("jobs.youtube_session.frequency_seconds") as number * 1000 || 50000);
+        konfigStore.get("jobs.youtube_session.frequency_seconds") as number *
+                1000 || 50000,
+    );
 } else if (innertubeClientOauthEnabled) {
     // Fired when waiting for the user to authorize the sign in attempt.
     innertubeClient.session.on("auth-pending", (data) => {
@@ -117,11 +114,11 @@ app.use("*", async (c, next) => {
 });
 
 routes(app, konfigStore);
+
 const https = Deno.env.get("HTTPS");
 const port = konfigStore.get("server.port") as number;
 const host = konfigStore.get("server.host") as string;
 
-
 if (https == "TRUE" || https == "true") {
     const cert = Deno.readTextFileSync("/data/cert.pem");
     const key = Deno.readTextFileSync("/data/key.key");

src/routes/health.ts

@@ -0,0 +1,12 @@
+import { Hono } from "hono";
+
+const health = new Hono();
+
+health.get("/", () => {
+  return new Response("OK", {
+	status: 200,
+    headers: { "Content-Type": "text/plain" },
+  });
+});
+
+export default health;

src/routes/index.ts

@@ -2,11 +2,73 @@ import { Hono } from "hono";
 import { logger } from "hono/logger";
 import { Store } from "@willsoto/node-konfig-core";
 import { bearerAuth } from "hono/bearer-auth";
+import { Registry, Counter, Gauge } from "prom-client"
 
 import youtubeApiPlayer from "./youtube_api_routes/player.ts";
 import invidiousRouteLatestVersion from "./invidious_routes/latestVersion.ts";
 import invidiousRouteDashManifest from "./invidious_routes/dashManifest.ts";
 import videoPlaybackProxy from "./videoPlaybackProxy.ts";
+import metrics from "./metrics.ts";
+
+const METRICS_PREFIX = "invidious_companion_";
+export const register = new Registry();
+
+export const externalTokenGeneratorFail = new Counter({
+	name: `${METRICS_PREFIX}externaltokengenerator_fail`,
+	help: 'TODO',
+	registers: [register]
+  });
+
+export const cachedEntries = new Gauge({
+	name: `${METRICS_PREFIX}cached_entries`,
+	help: 'TODO',
+	registers: [register]
+  });
+
+export const subreasonProtectCommunity = new Counter({
+	name: `${METRICS_PREFIX}subreason_protect_community`,
+	help: 'TODO',
+	registers: [register]
+  });
+
+export const poTokenFail = new Counter({
+	name: `${METRICS_PREFIX}potoken_fail`,
+	help: 'TODO',
+	registers: [register]
+  });
+
+export const reasonBot = new Counter({
+	name: `${METRICS_PREFIX}reason_bot`,
+	help: 'TODO',
+	registers: [register]
+  });
+
+export const videoUnavailable = new Counter({
+	name: `${METRICS_PREFIX}video_unavailable`,
+	help: 'TODO',
+	registers: [register]
+  });
+
+export const videoRestricted = new Counter({
+	name: `${METRICS_PREFIX}video_restricted`,
+	help: 'TODO',
+	registers: [register]
+  });
+
+
+export const failedRequests = new Counter({
+	name: `${METRICS_PREFIX}failed_requests`,
+	help: 'TODO',
+	registers: [register]
+});
+
+export const successfulRequests = new Counter({
+	name: `${METRICS_PREFIX}successful_requests`,
+	help: 'TODO',
+	registers: [register]
+});
+
+import health from "./health.ts";
 
 export const routes = (app: Hono, konfigStore: Store<Record<string, unknown>>) => {
   app.use("*", logger());
@@ -22,4 +84,6 @@ export const routes = (app: Hono, konfigStore: Store<Record<string, unknown>>) =
   app.route("/latest_version", invidiousRouteLatestVersion);
   app.route("/api/manifest/dash/id", invidiousRouteDashManifest);
   app.route("/videoplayback", videoPlaybackProxy);
+  app.route("/metrics", metrics)
+  app.route("/healthz", health);
 };

src/routes/invidious_routes/dashManifest.ts

@@ -24,6 +24,8 @@ dashManifest.get("/:videoId", async (c) => {
         Record<string, unknown>
     >;
 
+    const maxDashResolution = Deno.env.get("SERVER_MAX_DASH_RESOLUTION") || konfigStore.get("server.max_dash_resolution") as number;
+
     if (konfigStore.get("server.verify_requests") && check == undefined) {
         throw new HTTPException(400, {
             res: new Response("No check ID."),
@@ -67,7 +69,12 @@ dashManifest.get("/:videoId", async (c) => {
                         ).includes("av01")
                     ) {
                         if (i.mime_type.includes("av01")) {
-                            return true;
+                            // @ts-ignore 'i.height' is possibly 'undefined'.
+                            if (i.height > maxDashResolution) {
+                                return false;
+                            } else {
+                                return true;
+                            }
                         } else {
                             return false;
                         }

src/routes/metrics.ts

@@ -0,0 +1,19 @@
+import { Hono } from "hono";
+import { register, cachedEntries } from "./index.ts";
+import { kv } from "../lib/helpers/youtubePlayerHandling.ts"
+
+const metrics = new Hono();
+  
+metrics.get("/", async () => {
+  let i = 0;
+  const entries = kv.list({ prefix: ["video_cache"] })
+  for await (const _ of entries) {
+    i += 1
+  }
+  cachedEntries.set(i)
+  return new Response(await register.metrics(), {
+    headers: { "Content-Type": "text/plain" },
+  });
+});
+
+export default metrics;

src/routes/videoPlaybackProxy.ts

@@ -52,10 +52,6 @@ videoPlaybackProxy.get("/", async (c) => {
     // deno-lint-ignore prefer-const
     let queryParams = new URLSearchParams(urlReq.search);
     queryParams.delete("host");
-    // alr parameter is only for WEB/HTML5 clients
-    if (client.includes("WEB")) {
-        queryParams.append("alr", "yes");
-    }
     if (rangeHeader) {
         queryParams.append(
             "range",

src/routes/youtube_api_routes/player.ts

@@ -3,9 +3,33 @@ import { youtubePlayerParsing } from "../../lib/helpers/youtubePlayerHandling.ts
 import { HonoVariables } from "../../lib/types/HonoVariables.ts";
 import { Innertube } from "youtubei.js";
 import { Store } from "@willsoto/node-konfig-core";
+import { reasonBot, subreasonProtectCommunity, videoRestricted, videoUnavailable } from "../index.ts";
 
 const player = new Hono<{ Variables: HonoVariables }>();
 
+const errors = [
+  {
+    // @ts-ignore: Property 'playabilityStatus' does not exist on type 'object'
+    check: (yt: object) => yt.playabilityStatus?.reason?.includes("Sign in to confirm you’re not a bot"),
+    action: () => reasonBot.inc(),
+  },
+  {
+    // @ts-ignore: Property 'playabilityStatus' does not exist on type 'object'
+    check: (yt: object) => yt.playabilityStatus?.errorScreen?.playerErrorMessageRenderer?.subreason?.runs?.[0]?.text?.includes("This helps protect our community"),
+    action: () => subreasonProtectCommunity.inc(),
+  },
+  {
+    // @ts-ignore: Property 'playabilityStatus' does not exist on type 'object'
+    check: (yt: object) => yt.playabilityStatus?.errorScreen?.playerErrorMessageRenderer?.subreason?.runs?.[0]?.text?.includes("Video unavailable"),
+    action: () => videoUnavailable.inc(),
+  },
+  {
+    // @ts-ignore: Property 'playabilityStatus' does not exist on type 'object'
+    check: (yt: object) => yt.playabilityStatus?.reason?.includes("This video is restricted"),
+    action: () => videoRestricted.inc(),
+  },
+];
+
 player.post("/player", async (c) => {
   const jsonReq = await c.req.json();
   const innertubeClient = await c.get("innertubeClient") as Innertube;
@@ -14,9 +38,13 @@ player.post("/player", async (c) => {
     Record<string, unknown>
   >;
   if (jsonReq.videoId) {
-    return c.json(
+    const yt = await youtubePlayerParsing(innertubeClient, jsonReq.videoId, konfigStore)
-      await youtubePlayerParsing(innertubeClient, jsonReq.videoId, konfigStore)
+    errors.forEach((error) => {
-    );
+      if (error.check(yt)) {
+        error.action()
+      }
+    })
+    return c.json(yt);
   }
 });