diff --git a/patches/0001-ci-update-deno-to-2.2.5.patch b/patches/0001-ci-update-deno-to-2.2.5.patch index f46c98e..2c2b629 100644 --- a/patches/0001-ci-update-deno-to-2.2.5.patch +++ b/patches/0001-ci-update-deno-to-2.2.5.patch @@ -1,7 +1,7 @@ -From dbef0b607e6c3a9e5694b07708e445fe26b43ccb Mon Sep 17 00:00:00 2001 +From ad4b5aca25433218a7e903acffece4ebf222127e Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 19:37:34 -0300 -Subject: [PATCH 01/12] ci: update deno to 2.2.5 +Subject: [PATCH 01/13] ci: update deno to 2.2.5 --- Dockerfile | 2 +- diff --git a/patches/0002-feat-add-support-for-an-external-videoplayback-proxy.patch b/patches/0002-feat-add-support-for-an-external-videoplayback-proxy.patch index 4d61d92..96df614 100644 --- a/patches/0002-feat-add-support-for-an-external-videoplayback-proxy.patch +++ b/patches/0002-feat-add-support-for-an-external-videoplayback-proxy.patch @@ -1,7 +1,7 @@ -From 2028045428f15eb7a9a21cd438b44c110fad6b2d Mon Sep 17 00:00:00 2001 +From 38a460ca36d462c7d4396a4a42266c318d56c505 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 18:44:10 -0300 -Subject: [PATCH 02/12] feat: add support for an external videoplayback proxy +Subject: [PATCH 02/13] feat: add support for an external videoplayback proxy --- config/config.example.toml | 1 + diff --git a/patches/0003-feat-report-the-external-videoplayback-proxy-via-inf.patch b/patches/0003-feat-report-the-external-videoplayback-proxy-via-inf.patch index 1c82c55..9ef09f3 100644 --- a/patches/0003-feat-report-the-external-videoplayback-proxy-via-inf.patch +++ b/patches/0003-feat-report-the-external-videoplayback-proxy-via-inf.patch @@ -1,7 +1,7 @@ -From 86e9007023a5aef96547e6de6883d096b2dd9a87 Mon Sep 17 00:00:00 2001 +From ca369efa1837eff09e782f2e2d363ee23d66a43e Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 18:52:53 -0300 -Subject: [PATCH 03/12] feat: report the external videoplayback proxy via /info +Subject: [PATCH 03/13] feat: report the external videoplayback proxy via /info endpoint --- diff --git a/patches/0004-feat-add-resolution-limit-on-DASH-streams-to-save-ba.patch b/patches/0004-feat-add-resolution-limit-on-DASH-streams-to-save-ba.patch index 016418e..78e8c40 100644 --- a/patches/0004-feat-add-resolution-limit-on-DASH-streams-to-save-ba.patch +++ b/patches/0004-feat-add-resolution-limit-on-DASH-streams-to-save-ba.patch @@ -1,7 +1,7 @@ -From 6f4763821259cd0be12a63fec8c542d2c158bdac Mon Sep 17 00:00:00 2001 +From cd276db4eee8885f0022dc5a72ea069de8b54fd1 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 19:02:01 -0300 -Subject: [PATCH 04/12] feat: add resolution limit on DASH streams to save +Subject: [PATCH 04/13] feat: add resolution limit on DASH streams to save bandwidth --- diff --git a/patches/0005-feat-add-env-variable-to-set-verify_requests.patch b/patches/0005-feat-add-env-variable-to-set-verify_requests.patch index 3cc779f..11140eb 100644 --- a/patches/0005-feat-add-env-variable-to-set-verify_requests.patch +++ b/patches/0005-feat-add-env-variable-to-set-verify_requests.patch @@ -1,7 +1,7 @@ -From 93c942bdf866fc312955c0767ebd65eb0fee4ea9 Mon Sep 17 00:00:00 2001 +From 15a903e7478366173e4ac90f1eb96526fc94df8d Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 19:06:04 -0300 -Subject: [PATCH 05/12] feat: add env variable to set verify_requests +Subject: [PATCH 05/13] feat: add env variable to set verify_requests --- src/lib/helpers/config.ts | 4 +++- diff --git a/patches/0006-feat-add-support-for-multiple-proxies.patch b/patches/0006-feat-add-support-for-multiple-proxies.patch index 6630897..c835a10 100644 --- a/patches/0006-feat-add-support-for-multiple-proxies.patch +++ b/patches/0006-feat-add-support-for-multiple-proxies.patch @@ -1,7 +1,7 @@ -From eb60f19d368e37e77e01855fa643d19fb02246e4 Mon Sep 17 00:00:00 2001 +From d2fd6b4cc2aa123b82ba0c11a5d4593ca8cdc4dc Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 19:20:52 -0300 -Subject: [PATCH 06/12] feat: add support for multiple proxies +Subject: [PATCH 06/13] feat: add support for multiple proxies --- src/lib/helpers/getFetchClient.ts | 17 ++++++++++++++++- diff --git a/patches/0007-feat-add-option-to-disable-potoken-generation-check.patch b/patches/0007-feat-add-option-to-disable-potoken-generation-check.patch index c37dc29..025e0bb 100644 --- a/patches/0007-feat-add-option-to-disable-potoken-generation-check.patch +++ b/patches/0007-feat-add-option-to-disable-potoken-generation-check.patch @@ -1,7 +1,7 @@ -From 73d41d7b22d267d0f361bd7ad25658bde66b85b7 Mon Sep 17 00:00:00 2001 +From 600734d4b7731ad70605770cf99060d5af382647 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 20:34:33 -0300 -Subject: [PATCH 07/12] feat: add option to disable potoken generation check +Subject: [PATCH 07/13] feat: add option to disable potoken generation check --- config/config.example.toml | 1 + diff --git a/patches/0008-add-proxy-retries-on-innertube-error.patch b/patches/0008-add-proxy-retries-on-innertube-error.patch index 8f96c18..f6b2bfe 100644 --- a/patches/0008-add-proxy-retries-on-innertube-error.patch +++ b/patches/0008-add-proxy-retries-on-innertube-error.patch @@ -1,7 +1,7 @@ -From 05804a9205a7f865f32da0bc2466c1960bf976d7 Mon Sep 17 00:00:00 2001 +From a7672c1dba33fceb9c4de722a2ca6b2f56767007 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Tue, 25 Mar 2025 00:04:47 -0300 -Subject: [PATCH 08/12] add proxy retries on innertube error +Subject: [PATCH 08/13] add proxy retries on innertube error --- src/lib/helpers/config.ts | 1 + diff --git a/patches/0009-add-support-for-prometheus-metrics.patch b/patches/0009-add-support-for-prometheus-metrics.patch index c6c2d52..1446673 100644 --- a/patches/0009-add-support-for-prometheus-metrics.patch +++ b/patches/0009-add-support-for-prometheus-metrics.patch @@ -1,7 +1,7 @@ -From faa83d246d24bd480b443b21bc9acc528ca3932c Mon Sep 17 00:00:00 2001 +From 83b100af452ffe8b9892280c18667b57628b6f91 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Tue, 18 Mar 2025 16:38:23 -0300 -Subject: [PATCH 09/12] add support for prometheus metrics +Subject: [PATCH 09/13] add support for prometheus metrics fix deno lint and typo diff --git a/patches/0010-add-metrics-for-proxy-retries.patch b/patches/0010-add-metrics-for-proxy-retries.patch index dd86efb..00df869 100644 --- a/patches/0010-add-metrics-for-proxy-retries.patch +++ b/patches/0010-add-metrics-for-proxy-retries.patch @@ -1,7 +1,7 @@ -From 94a35c9561e990474daa959f73d7e6bab6be8ba2 Mon Sep 17 00:00:00 2001 +From dc1df7a313083757fd5b74da3dc1739b0e45eb27 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Tue, 25 Mar 2025 00:07:28 -0300 -Subject: [PATCH 10/12] add metrics for proxy retries +Subject: [PATCH 10/13] add metrics for proxy retries --- src/lib/helpers/metrics.ts | 5 +++++ diff --git a/patches/0011-fix-fix-tokio-overflow-on-compile.patch b/patches/0011-fix-fix-tokio-overflow-on-compile.patch index 0c74f51..4601635 100644 --- a/patches/0011-fix-fix-tokio-overflow-on-compile.patch +++ b/patches/0011-fix-fix-tokio-overflow-on-compile.patch @@ -1,7 +1,7 @@ -From 3fff970433b02e646cf1bc708402750c8849ac41 Mon Sep 17 00:00:00 2001 +From a9cc6d6dc6953ec1ea5bd49cb80e78ed25b6e0af Mon Sep 17 00:00:00 2001 From: Fijxu Date: Tue, 25 Mar 2025 00:24:07 -0300 -Subject: [PATCH 11/12] fix: fix tokio overflow on compile +Subject: [PATCH 11/13] fix: fix tokio overflow on compile --- Dockerfile | 2 ++ diff --git a/patches/0012-Add-environment-variable-for-youtube_session.frequen.patch b/patches/0012-Add-environment-variable-for-youtube_session.frequen.patch index eb0ad11..e0aee20 100644 --- a/patches/0012-Add-environment-variable-for-youtube_session.frequen.patch +++ b/patches/0012-Add-environment-variable-for-youtube_session.frequen.patch @@ -1,7 +1,7 @@ -From 1a9ce4f8ddb861e7bb77e50070101c5ef8ff2181 Mon Sep 17 00:00:00 2001 +From 046aa5c93e998ceaba089ab38b6ee5ed7162705c Mon Sep 17 00:00:00 2001 From: Fijxu Date: Wed, 26 Mar 2025 12:24:49 -0300 -Subject: [PATCH 12/12] Add environment variable for youtube_session.frequency +Subject: [PATCH 12/13] Add environment variable for youtube_session.frequency --- src/lib/helpers/config.ts | 4 +++- diff --git a/patches/0013-Move-po-token-to-webworker.patch b/patches/0013-Move-po-token-to-webworker.patch new file mode 100644 index 0000000..26c16cc --- /dev/null +++ b/patches/0013-Move-po-token-to-webworker.patch @@ -0,0 +1,699 @@ +From 9d4ad948d08584a771995bf06ddf209fc8c97ee7 Mon Sep 17 00:00:00 2001 +From: Fijxu +Date: Tue, 1 Apr 2025 19:18:15 -0300 +Subject: [PATCH 13/13] Move po token to webworker + +https://github.com/iv-org/invidious-companion/pull/73 + +Squashed commit of the following: + +commit 99929b0e8d1e26ca872437e0f1799efa3632dc28 +Author: Alex Maras +Date: Wed Mar 26 21:43:25 2025 +0800 + + chore: remove memory logging + +commit 12f7eee268cbc4224b6e2778d7e3abbbac9ae011 +Author: Alex Maras +Date: Mon Mar 24 18:14:40 2025 +0800 + + chore: deno fmt + +commit 9befe1b2bf6d2294b1765213bb5be98d649cce06 +Author: Alex Maras +Date: Mon Mar 24 18:13:25 2025 +0800 + + chore: improve typing in worker.ts + +commit 186efd73c05980b91c8e81290327cf5f8eaa1629 +Author: Alex Maras +Date: Sat Mar 22 10:57:43 2025 +0800 + + chore: fmt + +commit 391f91571a212d46055e35edfaa00b8efc2b5bc1 +Author: Alex Maras +Date: Sat Mar 22 10:57:08 2025 +0800 + + chore: use z.union instead of .or in worker + +commit 3df53068293b513b6502f0bbb29d549027411c5a +Author: Alex Maras +Date: Thu Mar 20 14:06:52 2025 +0800 + + chore: temporary heap memory usage output + +commit a1cb9da1ca3cd8890007d9b2ff4b2b400860228a +Author: Alex Maras +Date: Thu Mar 20 14:06:24 2025 +0800 + + feat: split po token generation to web worker +--- + src/lib/helpers/config.ts | 2 +- + src/lib/helpers/youtubePlayerHandling.ts | 4 +- + src/lib/helpers/youtubePlayerReq.ts | 6 +- + src/lib/jobs/potoken.ts | 225 +++++++++++----------- + src/lib/jobs/worker.ts | 231 +++++++++++++++++++++++ + src/lib/types/HonoVariables.ts | 4 +- + src/main.ts | 22 +-- + 7 files changed, 361 insertions(+), 133 deletions(-) + create mode 100644 src/lib/jobs/worker.ts + +diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts +index 8cf34fd..2a3fdb3 100644 +--- a/src/lib/helpers/config.ts ++++ b/src/lib/helpers/config.ts +@@ -1,7 +1,7 @@ + import { z, ZodError } from "zod"; + import { parse } from "@std/toml"; + +-const ConfigSchema = z.object({ ++export const ConfigSchema = z.object({ + server: z.object({ + port: z.number().default(Number(Deno.env.get("PORT")) || 8282), + host: z.string().default(Deno.env.get("HOST") || "127.0.0.1"), +diff --git a/src/lib/helpers/youtubePlayerHandling.ts b/src/lib/helpers/youtubePlayerHandling.ts +index 7b5f0c1..62eb30c 100644 +--- a/src/lib/helpers/youtubePlayerHandling.ts ++++ b/src/lib/helpers/youtubePlayerHandling.ts +@@ -1,8 +1,8 @@ + import { ApiResponse, Innertube, YT } from "youtubei.js"; + import { generateRandomString } from "youtubei.js/Utils"; + import { compress, decompress } from "brotli"; +-import type { BG } from "bgutils"; + import { Metrics } from "../helpers/metrics.ts"; ++import type { TokenMinter } from "../jobs/potoken.ts"; + let youtubePlayerReqLocation = "youtubePlayerReq"; + if (Deno.env.get("YT_PLAYER_REQ_LOCATION")) { + if (Deno.env.has("DENO_COMPILED")) { +@@ -31,7 +31,7 @@ export const youtubePlayerParsing = async ({ + innertubeClient: Innertube; + videoId: string; + config: Config; +- tokenMinter: BG.WebPoMinter; ++ tokenMinter: TokenMinter; + metrics: Metrics | undefined; + overrideCache?: boolean; + }): Promise => { +diff --git a/src/lib/helpers/youtubePlayerReq.ts b/src/lib/helpers/youtubePlayerReq.ts +index 884fae4..af78269 100644 +--- a/src/lib/helpers/youtubePlayerReq.ts ++++ b/src/lib/helpers/youtubePlayerReq.ts +@@ -1,6 +1,6 @@ + import { ApiResponse, Innertube } from "youtubei.js"; + import NavigationEndpoint from "youtubei.js/NavigationEndpoint"; +-import type { BG } from "bgutils"; ++import type { TokenMinter } from "../jobs/potoken.ts"; + + import type { Config } from "./config.ts"; + +@@ -8,7 +8,7 @@ export const youtubePlayerReq = async ( + innertubeClient: Innertube, + videoId: string, + config: Config, +- tokenMinter: BG.WebPoMinter, ++ tokenMinter: TokenMinter, + ): Promise => { + const innertubeClientOauthEnabled = config.youtube_session.oauth_enabled; + +@@ -21,7 +21,7 @@ export const youtubePlayerReq = async ( + watchEndpoint: { videoId: videoId }, + }); + +- const contentPoToken = await tokenMinter.mintAsWebsafeString(videoId); ++ const contentPoToken = await tokenMinter(videoId); + + return watch_endpoint.call(innertubeClient.actions, { + playbackContext: { +diff --git a/src/lib/jobs/potoken.ts b/src/lib/jobs/potoken.ts +index e7da3c0..fe99e9c 100644 +--- a/src/lib/jobs/potoken.ts ++++ b/src/lib/jobs/potoken.ts +@@ -1,6 +1,3 @@ +-import { BG, buildURL, GOOG_API_KEY, USER_AGENT } from "bgutils"; +-import type { WebPoSignalOutput } from "bgutils"; +-import { JSDOM } from "jsdom"; + import { Innertube, UniversalCache } from "youtubei.js"; + import { + youtubePlayerParsing, +@@ -21,121 +18,134 @@ if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) { + } + const { getFetchClient } = await import(getFetchClientLocation); + +-// Adapted from https://github.com/LuanRT/BgUtils/blob/main/examples/node/index.ts +-export const poTokenGenerate = async ( +- innertubeClient: Innertube, +- config: Config, +- innertubeClientCache: UniversalCache, +- metrics: Metrics | undefined, +-): Promise<{ innertubeClient: Innertube; tokenMinter: BG.WebPoMinter }> => { +- if (innertubeClient.session.po_token) { +- innertubeClient = await Innertube.create({ +- enable_session_cache: false, +- user_agent: USER_AGENT, +- retrieve_player: false, +- }); +- } ++import { InputMessage, OutputMessageSchema } from "./worker.ts"; + +- const fetchImpl = await getFetchClient(config); +- +- const visitorData = innertubeClient.session.context.client.visitorData; +- +- if (!visitorData) { +- throw new Error("Could not get visitor data"); +- } +- +- const dom = new JSDOM( +- '', +- { +- url: "https://www.youtube.com/", +- referrer: "https://www.youtube.com/", +- userAgent: USER_AGENT, +- }, +- ); +- +- Object.assign(globalThis, { +- window: dom.window, +- document: dom.window.document, +- location: dom.window.location, +- origin: dom.window.origin, +- }); ++interface TokenGeneratorWorker extends Omit { ++ postMessage(message: InputMessage): void; ++} + +- if (!Reflect.has(globalThis, "navigator")) { +- Object.defineProperty(globalThis, "navigator", { +- value: dom.window.navigator, ++const workers: TokenGeneratorWorker[] = []; ++ ++function createMinter(worker: TokenGeneratorWorker) { ++ return (videoId: string): Promise => { ++ const { promise, resolve } = Promise.withResolvers(); ++ // generate a UUID to identify the request as many minter calls ++ // may be made within a timespan, and this function will be ++ // informed about all of them until it's got its own ++ const requestId = crypto.randomUUID(); ++ const listener = (message: MessageEvent) => { ++ const parsedMessage = OutputMessageSchema.parse(message.data); ++ if ( ++ parsedMessage.type === "content-token" && ++ parsedMessage.requestId === requestId ++ ) { ++ worker.removeEventListener("message", listener); ++ resolve(parsedMessage.contentToken); ++ } ++ }; ++ worker.addEventListener("message", listener); ++ worker.postMessage({ ++ type: "content-token-request", ++ videoId, ++ requestId, + }); +- } +- +- const challengeResponse = await innertubeClient.getAttestationChallenge( +- "ENGAGEMENT_TYPE_UNBOUND", +- ); +- if (!challengeResponse.bg_challenge) { +- throw new Error("Could not get challenge"); +- } +- +- const interpreterUrl = challengeResponse.bg_challenge.interpreter_url +- .private_do_not_access_or_else_trusted_resource_url_wrapped_value; +- const bgScriptResponse = await fetchImpl( +- `http:${interpreterUrl}`, +- ); +- const interpreterJavascript = await bgScriptResponse.text(); + +- if (interpreterJavascript) { +- new Function(interpreterJavascript)(); +- } else throw new Error("Could not load VM"); ++ return promise; ++ }; ++} + +- // Botguard currently surfaces a "Not implemented" error here, due to the environment +- // not having a valid Canvas API in JSDOM. At the time of writing, this doesn't cause +- // any issues as the Canvas check doesn't appear to be an enforced element of the checks +- console.log( +- '[INFO] the "Not implemented: HTMLCanvasElement.prototype.getContext" error is normal. Please do not open a bug report about it.', +- ); +- const botguard = await BG.BotGuardClient.create({ +- program: challengeResponse.bg_challenge.program, +- globalName: challengeResponse.bg_challenge.global_name, +- globalObj: globalThis, +- }); ++export type TokenMinter = ReturnType; + +- const webPoSignalOutput: WebPoSignalOutput = []; +- const botguardResponse = await botguard.snapshot({ webPoSignalOutput }); +- const requestKey = "O43z0dpjhgX20SCx4KAo"; ++// Adapted from https://github.com/LuanRT/BgUtils/blob/main/examples/node/index.ts ++export const poTokenGenerate = ( ++ config: Config, ++ innertubeClientCache: UniversalCache, ++ metrics: Metrics | undefined, ++): Promise<{ innertubeClient: Innertube; tokenMinter: TokenMinter }> => { ++ const { promise, resolve, reject } = Promise.withResolvers< ++ Awaited> ++ >(); + +- const integrityTokenResponse = await fetchImpl( +- buildURL("GenerateIT", true), ++ const worker: TokenGeneratorWorker = new Worker( ++ new URL("./worker.ts", import.meta.url).href, + { +- method: "POST", +- headers: { +- "content-type": "application/json+protobuf", +- "x-goog-api-key": GOOG_API_KEY, +- "x-user-agent": "grpc-web-javascript/0.1", +- "user-agent": USER_AGENT, +- }, +- body: JSON.stringify([requestKey, botguardResponse]), ++ type: "module", ++ name: "PO Token Generator", + }, + ); ++ // take note of the worker so we can kill it once a new one takes its place ++ workers.push(worker); ++ worker.addEventListener("message", async (event) => { ++ const parsedMessage = OutputMessageSchema.parse(event.data); ++ ++ // worker is listening for messages ++ if (parsedMessage.type === "ready") { ++ const untypedPostMessage = worker.postMessage.bind(worker); ++ worker.postMessage = (message: InputMessage) => ++ untypedPostMessage(message); ++ worker.postMessage({ type: "initialise", config }); ++ } + +- const response = await integrityTokenResponse.json() as unknown[]; +- +- if (typeof response[0] !== "string") { +- throw new Error("Could not get integrity token"); +- } ++ if (parsedMessage.type === "error") { ++ console.log({ errorFromWorker: parsedMessage.error }); ++ worker.terminate(); ++ reject(parsedMessage.error); ++ } + +- const integrityTokenBasedMinter = await BG.WebPoMinter.create({ +- integrityToken: response[0], +- }, webPoSignalOutput); ++ // worker is initialised and has passed back a session token and visitor data ++ if (parsedMessage.type === "initialised") { ++ try { ++ const instantiatedInnertubeClient = await Innertube.create({ ++ enable_session_cache: false, ++ po_token: parsedMessage.sessionPoToken, ++ visitor_data: parsedMessage.visitorData, ++ fetch: getFetchClient(config), ++ cache: innertubeClientCache, ++ generate_session_locally: true, ++ }); ++ const minter = createMinter(worker); ++ // check token from minter ++ await checkToken({ ++ instantiatedInnertubeClient, ++ config, ++ integrityTokenBasedMinter: minter, ++ metrics, ++ }); ++ console.log("Successfully generated PO token"); ++ const numberToKill = workers.length - 1; ++ for (let i = 0; i < numberToKill; i++) { ++ const workerToKill = workers.shift(); ++ workerToKill?.terminate(); ++ } ++ return resolve({ ++ innertubeClient: instantiatedInnertubeClient, ++ tokenMinter: minter, ++ }); ++ } catch (err) { ++ console.log("Failed to get valid PO token, will retry", { ++ err, ++ }); ++ worker.terminate(); ++ reject(err); ++ } ++ } ++ }); + +- const sessionPoToken = await integrityTokenBasedMinter.mintAsWebsafeString( +- visitorData, +- ); ++ return promise; ++}; + +- const instantiatedInnertubeClient = await Innertube.create({ +- enable_session_cache: false, +- po_token: sessionPoToken, +- visitor_data: visitorData, +- fetch: getFetchClient(config), +- cache: innertubeClientCache, +- generate_session_locally: true, +- }); ++async function checkToken({ ++ instantiatedInnertubeClient, ++ config, ++ integrityTokenBasedMinter, ++ metrics, ++}: { ++ instantiatedInnertubeClient: Innertube; ++ config: Config; ++ integrityTokenBasedMinter: TokenMinter; ++ metrics: Metrics | undefined, ++}) { ++ const fetchImpl = getFetchClient(config); + + if (config.jobs.youtube_session.po_token_check) { + try { +@@ -183,9 +193,4 @@ export const poTokenGenerate = async ( + throw err; + } + } +- +- return { +- innertubeClient: instantiatedInnertubeClient, +- tokenMinter: integrityTokenBasedMinter, +- }; +-}; ++} +diff --git a/src/lib/jobs/worker.ts b/src/lib/jobs/worker.ts +new file mode 100644 +index 0000000..0d62bfc +--- /dev/null ++++ b/src/lib/jobs/worker.ts +@@ -0,0 +1,231 @@ ++/// ++ ++import { z } from "zod"; ++import { Config, ConfigSchema } from "../helpers/config.ts"; ++import { BG, buildURL, GOOG_API_KEY, USER_AGENT } from "bgutils"; ++import type { WebPoSignalOutput } from "bgutils"; ++import { JSDOM } from "jsdom"; ++import { Innertube } from "youtubei.js"; ++let getFetchClientLocation = "getFetchClient"; ++if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) { ++ if (Deno.env.has("DENO_COMPILED")) { ++ getFetchClientLocation = Deno.mainModule.replace("src/main.ts", "") + ++ Deno.env.get("GET_FETCH_CLIENT_LOCATION"); ++ } else { ++ getFetchClientLocation = Deno.env.get( ++ "GET_FETCH_CLIENT_LOCATION", ++ ) as string; ++ } ++} ++ ++type FetchFunction = typeof fetch; ++const { getFetchClient }: { ++ getFetchClient: (config: Config) => Promise; ++} = await import(getFetchClientLocation); ++ ++// ---- Messages to send to the webworker ---- ++const InputInitialiseSchema = z.object({ ++ type: z.literal("initialise"), ++ config: ConfigSchema, ++}).strict(); ++ ++const InputContentTokenSchema = z.object({ ++ type: z.literal("content-token-request"), ++ videoId: z.string(), ++ requestId: z.string().uuid(), ++}).strict(); ++export type InputInitialise = z.infer; ++export type InputContentToken = z.infer; ++const InputMessageSchema = z.union([ ++ InputInitialiseSchema, ++ InputContentTokenSchema, ++]); ++export type InputMessage = z.infer; ++ ++// ---- Messages that the webworker sends to the parent ---- ++const OutputReadySchema = z.object({ ++ type: z.literal("ready"), ++}).strict(); ++ ++const OutputInitialiseSchema = z.object({ ++ type: z.literal("initialised"), ++ sessionPoToken: z.string(), ++ visitorData: z.string(), ++}).strict(); ++ ++const OutputContentTokenSchema = z.object({ ++ type: z.literal("content-token"), ++ contentToken: z.string(), ++ requestId: InputContentTokenSchema.shape.requestId, ++}).strict(); ++ ++const OutputErrorSchema = z.object({ ++ type: z.literal("error"), ++ error: z.any(), ++}).strict(); ++export const OutputMessageSchema = z.union([ ++ OutputReadySchema, ++ OutputInitialiseSchema, ++ OutputContentTokenSchema, ++ OutputErrorSchema, ++]); ++type OutputMessage = z.infer; ++ ++const IntegrityTokenResponse = z.tuple([z.string()]).rest(z.any()); ++ ++const isWorker = typeof WorkerGlobalScope !== "undefined" && ++ self instanceof WorkerGlobalScope; ++if (isWorker) { ++ // helper function to force type-checking ++ const untypedPostmessage = self.postMessage.bind(self); ++ const postMessage = (message: OutputMessage) => { ++ untypedPostmessage(message); ++ }; ++ ++ let minter: BG.WebPoMinter; ++ ++ onmessage = async (event) => { ++ const message = InputMessageSchema.parse(event.data); ++ if (message.type === "initialise") { ++ const fetchImpl: typeof fetch = await getFetchClient( ++ message.config, ++ ); ++ try { ++ const { ++ sessionPoToken, ++ visitorData, ++ generatedMinter, ++ } = await setup({ fetchImpl }); ++ minter = generatedMinter; ++ postMessage({ ++ type: "initialised", ++ sessionPoToken, ++ visitorData, ++ }); ++ } catch (err) { ++ postMessage({ type: "error", error: err }); ++ } ++ } ++ // this is called every time a video needs a content token ++ if (message.type === "content-token-request") { ++ if (!minter) { ++ throw new Error( ++ "Minter not yet ready, must initialise first", ++ ); ++ } ++ const contentToken = await minter.mintAsWebsafeString( ++ message.videoId, ++ ); ++ postMessage({ ++ type: "content-token", ++ contentToken, ++ requestId: message.requestId, ++ }); ++ } ++ }; ++ ++ postMessage({ type: "ready" }); ++} ++ ++async function setup( ++ { fetchImpl }: { fetchImpl: FetchFunction }, ++) { ++ const innertubeClient = await Innertube.create({ ++ enable_session_cache: false, ++ user_agent: USER_AGENT, ++ retrieve_player: false, ++ }); ++ ++ const visitorData = innertubeClient.session.context.client.visitorData; ++ ++ if (!visitorData) { ++ throw new Error("Could not get visitor data"); ++ } ++ ++ const dom = new JSDOM( ++ '', ++ { ++ url: "https://www.youtube.com/", ++ referrer: "https://www.youtube.com/", ++ userAgent: USER_AGENT, ++ }, ++ ); ++ ++ Object.assign(globalThis, { ++ window: dom.window, ++ document: dom.window.document, ++ // location: dom.window.location, // --- doesn't seem to be necessary and the Web Worker doesn't like it ++ origin: dom.window.origin, ++ }); ++ ++ if (!Reflect.has(globalThis, "navigator")) { ++ Object.defineProperty(globalThis, "navigator", { ++ value: dom.window.navigator, ++ }); ++ } ++ ++ const challengeResponse = await innertubeClient.getAttestationChallenge( ++ "ENGAGEMENT_TYPE_UNBOUND", ++ ); ++ if (!challengeResponse.bg_challenge) { ++ throw new Error("Could not get challenge"); ++ } ++ ++ const interpreterUrl = challengeResponse.bg_challenge.interpreter_url ++ .private_do_not_access_or_else_trusted_resource_url_wrapped_value; ++ const bgScriptResponse = await fetchImpl( ++ `https:${interpreterUrl}`, ++ ); ++ const interpreterJavascript = await bgScriptResponse.text(); ++ ++ if (interpreterJavascript) { ++ new Function(interpreterJavascript)(); ++ } else throw new Error("Could not load VM"); ++ ++ // Botguard currently surfaces a "Not implemented" error here, due to the environment ++ // not having a valid Canvas API in JSDOM. At the time of writing, this doesn't cause ++ // any issues as the Canvas check doesn't appear to be an enforced element of the checks ++ console.log( ++ '[INFO] the "Not implemented: HTMLCanvasElement.prototype.getContext" error is normal. Please do not open a bug report about it.', ++ ); ++ const botguard = await BG.BotGuardClient.create({ ++ program: challengeResponse.bg_challenge.program, ++ globalName: challengeResponse.bg_challenge.global_name, ++ globalObj: globalThis, ++ }); ++ ++ const webPoSignalOutput: WebPoSignalOutput = []; ++ const botguardResponse = await botguard.snapshot({ webPoSignalOutput }); ++ const requestKey = "O43z0dpjhgX20SCx4KAo"; ++ ++ const integrityTokenResponse = await fetchImpl( ++ buildURL("GenerateIT", true), ++ { ++ method: "POST", ++ headers: { ++ "content-type": "application/json+protobuf", ++ "x-goog-api-key": GOOG_API_KEY, ++ "x-user-agent": "grpc-web-javascript/0.1", ++ "user-agent": USER_AGENT, ++ }, ++ body: JSON.stringify([requestKey, botguardResponse]), ++ }, ++ ); ++ const integrityTokenBody = IntegrityTokenResponse.parse( ++ await integrityTokenResponse.json(), ++ ); ++ ++ const integrityTokenBasedMinter = await BG.WebPoMinter.create({ ++ integrityToken: integrityTokenBody[0], ++ }, webPoSignalOutput); ++ ++ const sessionPoToken = await integrityTokenBasedMinter.mintAsWebsafeString( ++ visitorData, ++ ); ++ ++ return { ++ sessionPoToken, ++ visitorData, ++ generatedMinter: integrityTokenBasedMinter, ++ }; ++} +diff --git a/src/lib/types/HonoVariables.ts b/src/lib/types/HonoVariables.ts +index 1e9b7a2..ab385ab 100644 +--- a/src/lib/types/HonoVariables.ts ++++ b/src/lib/types/HonoVariables.ts +@@ -1,11 +1,11 @@ + import { Innertube } from "youtubei.js"; +-import { BG } from "bgutils"; ++import type { TokenMinter } from "../jobs/potoken.ts"; + import type { Config } from "../helpers/config.ts"; + import { Metrics } from "../helpers/metrics.ts"; + + export type HonoVariables = { + innertubeClient: Innertube; + config: Config; +- tokenMinter: BG.WebPoMinter; ++ tokenMinter: TokenMinter; + metrics: Metrics | undefined; + }; +diff --git a/src/main.ts b/src/main.ts +index 851cd71..cc9e97d 100644 +--- a/src/main.ts ++++ b/src/main.ts +@@ -1,10 +1,9 @@ + import { Hono } from "hono"; + import { routes } from "./routes/index.ts"; + import { Innertube, UniversalCache } from "youtubei.js"; +-import { poTokenGenerate } from "./lib/jobs/potoken.ts"; ++import { poTokenGenerate, type TokenMinter } from "./lib/jobs/potoken.ts"; + import { USER_AGENT } from "bgutils"; + import { retry } from "@std/async"; +-import type { BG } from "bgutils"; + import type { HonoVariables } from "./lib/types/HonoVariables.ts"; + + import { parseConfig } from "./lib/helpers/config.ts"; +@@ -30,7 +29,7 @@ declare module "hono" { + const app = new Hono(); + const metrics = config.server.enable_metrics ? new Metrics() : undefined; + +-let tokenMinter: BG.WebPoMinter; ++let tokenMinter: TokenMinter; + let innertubeClient: Innertube; + let innertubeClientFetchPlayer = true; + const innertubeClientOauthEnabled = config.youtube_session.oauth_enabled; +@@ -65,7 +64,6 @@ if (!innertubeClientOauthEnabled) { + ({ innertubeClient, tokenMinter } = await retry( + poTokenGenerate.bind( + poTokenGenerate, +- innertubeClient, + config, + innertubeClientCache, + metrics, +@@ -79,17 +77,11 @@ if (!innertubeClientOauthEnabled) { + { backoffSchedule: [5_000, 15_000, 60_000, 180_000] }, + async () => { + if (innertubeClientJobPoTokenEnabled) { +- try { +- ({ innertubeClient, tokenMinter } = await poTokenGenerate( +- innertubeClient, +- config, +- innertubeClientCache, +- metrics, +- )); +- } catch (err) { +- metrics?.potokenGenerationFailure.inc(); +- throw err; +- } ++ ({ innertubeClient, tokenMinter } = await poTokenGenerate( ++ config, ++ innertubeClientCache, ++ metrics, ++ )); + } else { + innertubeClient = await Innertube.create({ + enable_session_cache: false, +-- +2.49.0 +