Merge remote-tracking branch 'upstream/master'

Dockerfile

@@ -48,3 +48,5 @@ WORKDIR /app
 USER appuser
 
 ENTRYPOINT ["/tini", "--", "/app/invidious_companion"]
+
+HEALTHCHECK --interval=5s --timeout=5s --start-period=10s --retries=3 CMD ["/tini", "--", "/app/invidious_companion", "healthcheck"]

config/default.toml

@@ -19,6 +19,11 @@ proxy_file = "proxies.txt"
 # Enable YouTube new video format UMP
 ump = false
 # external_videoplayback_proxy = ""
+# fetch_timeout_ms = 10000
+# fetch_retry_enable = true
+# fetch_retry_times = 3
+# fetch_retry_initial_debounce = 500
+# fetch_retry_debounce_multiplier = 2
 
 [jobs]
 

deno.json

@@ -8,9 +8,9 @@
     "hono/logger": "jsr:@hono/hono@^4.6.5/logger",
     "hono/bearer-auth": "jsr:@hono/hono@^4.6.5/bearer-auth",
     "prom-client": "npm:prom-client@^15.1.3",
-    "youtubei.js": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.0.0-deno/deno.ts",
+    "youtubei.js": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno.ts",
-    "youtubei.js/Utils": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.0.0-deno/deno/src/utils/Utils.ts",
+    "youtubei.js/Utils": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/utils/Utils.ts",
-    "youtubei.js/NavigationEndpoint": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.0.0-deno/deno/src/parser/classes/NavigationEndpoint.ts",
+    "youtubei.js/NavigationEndpoint": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/parser/classes/NavigationEndpoint.ts",
     "jsdom": "npm:jsdom@25.0.1",
     "bgutils": "https://esm.sh/bgutils-js@3.1.0",
     "estree": "https://esm.sh/@types/estree@1.0.6",

src/lib/helpers/getFetchClient.ts

@@ -1,4 +1,11 @@
 import { Store } from "@willsoto/node-konfig-core";
+import { retry, type RetryOptions } from "jsr:@std/async";
+
+type FetchInputParameter = Parameters<typeof fetch>[0];
+type FetchInitParameterWithClient =
+    | RequestInit
+    | RequestInit & { client: Deno.HttpClient };
+type FetchReturn = ReturnType<typeof fetch>;
 
 let proxies: string[] = [];
 let currentProxyIndex = 0;
@@ -15,21 +22,17 @@ try {
 }
 
 export const getFetchClient = (konfigStore: Store): {
-    (input: RequestInfo | URL, init?: RequestInit): Promise<Response>;
     (
-        input: Request | URL | string,
+        input: FetchInputParameter,
-        init?: RequestInit & {
+        init?: FetchInitParameterWithClient,
-            client: Deno.HttpClient;
+    ): FetchReturn;
-        },
-    ): Promise<Response>;
-    (input: URL | Request | string, init?: RequestInit): Promise<Response>;
 } => {
     if (
         Deno.env.get("PROXY") || konfigStore.get("networking.proxy") ||
         (proxies.length > 0)
     ) {
         return async (
-            input: RequestInfo | URL,
+            input: FetchInputParameter,
             init?: RequestInit,
         ) => {
             const proxyUrl = Deno.env.get("PROXY") ||
@@ -60,5 +63,39 @@ export const getFetchClient = (konfigStore: Store): {
         };
     }
 
-    return globalThis.fetch;
+    return (input: FetchInputParameter, init?: FetchInitParameterWithClient) =>
+        fetchShim(konfigStore, input, init);
 };
+
+function fetchShim(
+    konfigStore: Store,
+    input: FetchInputParameter,
+    init?: FetchInitParameterWithClient,
+): FetchReturn {
+    const fetchTimeout = konfigStore.get("networking.fetch_timeout_ms");
+    const fetchRetry = konfigStore.get("networking.fetch_retry_enable");
+    const fetchMaxAttempts = konfigStore.get("networking.fetch_retry_times");
+    const fetchInitialDebounce = konfigStore.get(
+        "networking.fetch_retry_initial_debounce",
+    );
+    const fetchDebounceMultiplier = konfigStore.get(
+        "networking.fetch_retry_debounce_multiplier",
+    );
+    const retryOptions: RetryOptions = {
+        maxAttempts: Number(fetchMaxAttempts) || 1,
+        minTimeout: Number(fetchInitialDebounce) || 0,
+        multiplier: Number(fetchDebounceMultiplier) || 0,
+        jitter: 0,
+    };
+
+    const callFetch = () =>
+        fetch(input, {
+            // only set the AbortSignal if the timeout is supplied in the config
+            signal: fetchTimeout
+                ? AbortSignal.timeout(Number(fetchTimeout))
+                : null,
+            ...(init || {}),
+        });
+    // if retry enabled, call retry with the fetch shim, otherwise pass the fetch shim back directly
+    return fetchRetry ? retry(callFetch, retryOptions) : callFetch();
+}

src/lib/types/HonoVariables.ts

@@ -1,5 +1,7 @@
 import { Innertube } from "youtubei.js";
+import type { konfigLoader } from "../helpers/konfigLoader.ts";
 
 export type HonoVariables = {
     innertubeClient: Innertube;
+    konfigStore: Awaited<ReturnType<typeof konfigLoader>>;
 };

src/main.ts

@@ -13,6 +13,7 @@ import { konfigLoader } from "./lib/helpers/konfigLoader.ts";
 import { ICache } from "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.0.0-deno/deno/src/types/Cache.ts";
 import { FetchFunction } from "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.0.0-deno/deno/src/types/PlatformShim.ts";
 import { getRandomUserAgent, PlayerError } from "youtubei.js/Utils";
+import type { HonoVariables } from "./lib/types/HonoVariables.ts";
 let getFetchClientLocation = "getFetchClient";
 if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
     if (Deno.env.has("DENO_COMPILED")) {
@@ -24,10 +25,31 @@ if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
         ) as string;
     }
 }
+
+const args = Deno.args;
+const konfigStore = await konfigLoader();
+const host = Deno.env.get("HOST") || konfigStore.get("server.host") as string;
+const port = Deno.env.get("PORT") || konfigStore.get("server.port") as string;
+
+if (args?.[0] == "healthcheck") {
+    try {
+        const response = await fetch(`http://${host}:${port}/healthz`);
+        if (response.status === 200) {
+            Deno.exit(0);
+        } else {
+            Deno.exit(1);
+        }
+    } catch (_) {
+        Deno.exit(1);
+    }
+}
+
 const { getFetchClient } = await import(getFetchClientLocation);
 
+declare module "hono" {
+    interface ContextVariableMap extends HonoVariables {}
+}
 const app = new Hono();
-const konfigStore = await konfigLoader();
 
 let innertubeClient: Innertube;
 let innertubeClientFetchPlayer = true;
@@ -215,9 +237,7 @@ if (!innertubeClientOauthEnabled) {
 }
 
 app.use("*", async (c, next) => {
-    // @ts-ignore Do not understand how to fix this error.
     c.set("innertubeClient", innertubeClient);
-    // @ts-ignore Do not understand how to fix this error.
     c.set("konfigStore", konfigStore);
     await next();
 });
@@ -225,8 +245,6 @@ app.use("*", async (c, next) => {
 routes(app, konfigStore);
 
 const https = Deno.env.get("HTTPS");
-const port = konfigStore.get("server.port") as number;
-const host = konfigStore.get("server.host") as string;
 
 if (https == "TRUE" || https == "true") {
     const cert = Deno.readTextFileSync("/data/cert.pem");

src/routes/invidious_routes/dashManifest.ts

@@ -1,7 +1,5 @@
 import { Hono } from "hono";
-import { FormatUtils, Innertube } from "youtubei.js";
+import { FormatUtils } from "youtubei.js";
-import { HonoVariables } from "../../lib/types/HonoVariables.ts";
-import { Store } from "@willsoto/node-konfig-core";
 import {
     youtubePlayerParsing,
     youtubeVideoInfo,
@@ -9,18 +7,15 @@ import {
 import { verifyRequest } from "../../lib/helpers/verifyRequest.ts";
 import { HTTPException } from "hono/http-exception";
 
-const dashManifest = new Hono<{ Variables: HonoVariables }>();
+const dashManifest = new Hono();
 
 dashManifest.get("/:videoId", async (c) => {
     const { videoId } = c.req.param();
     const { check, local } = c.req.query();
     c.header("access-control-allow-origin", "*");
 
-    const innertubeClient = await c.get("innertubeClient") as Innertube;
+    const innertubeClient = c.get("innertubeClient");
-    // @ts-ignore Do not understand how to fix this error.
+    const konfigStore = c.get("konfigStore");
-    const konfigStore = await c.get("konfigStore") as Store<
-        Record<string, unknown>
-    >;
 
     const maxDashResolution = Deno.env.get("SERVER_MAX_DASH_RESOLUTION") ||
         konfigStore.get("server.max_dash_resolution") as number;

src/routes/invidious_routes/latestVersion.ts

@@ -1,15 +1,12 @@
 import { Hono } from "hono";
 import { HTTPException } from "hono/http-exception";
-import { Innertube } from "youtubei.js";
-import { HonoVariables } from "../../lib/types/HonoVariables.ts";
-import { Store } from "@willsoto/node-konfig-core";
 import {
     youtubePlayerParsing,
     youtubeVideoInfo,
 } from "../../lib/helpers/youtubePlayerHandling.ts";
 import { verifyRequest } from "../../lib/helpers/verifyRequest.ts";
 
-const latestVersion = new Hono<{ Variables: HonoVariables }>();
+const latestVersion = new Hono();
 
 latestVersion.get("/", async (c) => {
     const { check, itag, id, local } = c.req.query();
@@ -21,19 +18,14 @@ latestVersion.get("/", async (c) => {
         });
     }
 
-    const innertubeClient = await c.get("innertubeClient") as Innertube;
+    const innertubeClient = c.get("innertubeClient");
-    // @ts-ignore Do not understand how to fix this error.
+    const konfigStore = await c.get("konfigStore");
-    const konfigStore = await c.get("konfigStore") as Store<
-        Record<string, unknown>
-    >;
-    const verifyRequests = Deno.env.get("VERIFY_REQUESTS") ||
-        konfigStore.get("server.verify_requests");
 
-    if (verifyRequests && check == undefined) {
+    if (konfigStore.get("server.verify_requests") && check == undefined) {
         throw new HTTPException(400, {
             res: new Response("No check ID."),
         });
-    } else if (verifyRequests && check) {
+    } else if (konfigStore.get("server.verify_requests") && check) {
         if (verifyRequest(check, id, konfigStore) === false) {
             throw new HTTPException(400, {
                 res: new Response("ID incorrect."),

src/routes/videoPlaybackProxy.ts

@@ -1,5 +1,4 @@
 import { Hono } from "hono";
-import { Store } from "@willsoto/node-konfig-core";
 import { HTTPException } from "hono/http-exception";
 let getFetchClientLocation = "getFetchClient";
 if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
@@ -44,10 +43,7 @@ videoPlaybackProxy.get("/", async (c) => {
         });
     }
 
-    // @ts-ignore Do not understand how to fix this error.
+    const konfigStore = c.get("konfigStore");
-    const konfigStore = await c.get("konfigStore") as Store<
-        Record<string, unknown>
-    >;
 
     // deno-lint-ignore prefer-const
     let queryParams = new URLSearchParams(urlReq.search);

src/routes/youtube_api_routes/player.ts

@@ -1,8 +1,5 @@
 import { Hono } from "hono";
 import { youtubePlayerParsing } from "../../lib/helpers/youtubePlayerHandling.ts";
-import { HonoVariables } from "../../lib/types/HonoVariables.ts";
-import { Innertube } from "youtubei.js";
-import { Store } from "@willsoto/node-konfig-core";
 import {
     reasonBot,
     subreasonProtectCommunity,
@@ -10,7 +7,7 @@ import {
     videoUnavailable,
 } from "metrics";
 
-const player = new Hono<{ Variables: HonoVariables }>();
+const player = new Hono();
 
 const errors = [
     {
@@ -47,11 +44,8 @@ const errors = [
 
 player.post("/player", async (c) => {
     const jsonReq = await c.req.json();
-    const innertubeClient = await c.get("innertubeClient") as Innertube;
+    const innertubeClient = c.get("innertubeClient");
-    // @ts-ignore Do not understand how to fix this error.
+    const konfigStore = c.get("konfigStore");
-    const konfigStore = await c.get("konfigStore") as Store<
-        Record<string, unknown>
-    >;
     if (jsonReq.videoId) {
         const yt = await youtubePlayerParsing(
             innertubeClient,