|
|
|
|
@ -0,0 +1,699 @@
|
|
|
|
|
From 9d4ad948d08584a771995bf06ddf209fc8c97ee7 Mon Sep 17 00:00:00 2001
|
|
|
|
|
From: Fijxu <fijxu@nadeko.net>
|
|
|
|
|
Date: Tue, 1 Apr 2025 19:18:15 -0300
|
|
|
|
|
Subject: [PATCH 13/13] Move po token to webworker
|
|
|
|
|
|
|
|
|
|
https://github.com/iv-org/invidious-companion/pull/73
|
|
|
|
|
|
|
|
|
|
Squashed commit of the following:
|
|
|
|
|
|
|
|
|
|
commit 99929b0e8d1e26ca872437e0f1799efa3632dc28
|
|
|
|
|
Author: Alex Maras <dev@alexmaras.com>
|
|
|
|
|
Date: Wed Mar 26 21:43:25 2025 +0800
|
|
|
|
|
|
|
|
|
|
chore: remove memory logging
|
|
|
|
|
|
|
|
|
|
commit 12f7eee268cbc4224b6e2778d7e3abbbac9ae011
|
|
|
|
|
Author: Alex Maras <dev@alexmaras.com>
|
|
|
|
|
Date: Mon Mar 24 18:14:40 2025 +0800
|
|
|
|
|
|
|
|
|
|
chore: deno fmt
|
|
|
|
|
|
|
|
|
|
commit 9befe1b2bf6d2294b1765213bb5be98d649cce06
|
|
|
|
|
Author: Alex Maras <dev@alexmaras.com>
|
|
|
|
|
Date: Mon Mar 24 18:13:25 2025 +0800
|
|
|
|
|
|
|
|
|
|
chore: improve typing in worker.ts
|
|
|
|
|
|
|
|
|
|
commit 186efd73c05980b91c8e81290327cf5f8eaa1629
|
|
|
|
|
Author: Alex Maras <dev@alexmaras.com>
|
|
|
|
|
Date: Sat Mar 22 10:57:43 2025 +0800
|
|
|
|
|
|
|
|
|
|
chore: fmt
|
|
|
|
|
|
|
|
|
|
commit 391f91571a212d46055e35edfaa00b8efc2b5bc1
|
|
|
|
|
Author: Alex Maras <dev@alexmaras.com>
|
|
|
|
|
Date: Sat Mar 22 10:57:08 2025 +0800
|
|
|
|
|
|
|
|
|
|
chore: use z.union instead of .or in worker
|
|
|
|
|
|
|
|
|
|
commit 3df53068293b513b6502f0bbb29d549027411c5a
|
|
|
|
|
Author: Alex Maras <dev@alexmaras.com>
|
|
|
|
|
Date: Thu Mar 20 14:06:52 2025 +0800
|
|
|
|
|
|
|
|
|
|
chore: temporary heap memory usage output
|
|
|
|
|
|
|
|
|
|
commit a1cb9da1ca3cd8890007d9b2ff4b2b400860228a
|
|
|
|
|
Author: Alex Maras <dev@alexmaras.com>
|
|
|
|
|
Date: Thu Mar 20 14:06:24 2025 +0800
|
|
|
|
|
|
|
|
|
|
feat: split po token generation to web worker
|
|
|
|
|
---
|
|
|
|
|
src/lib/helpers/config.ts | 2 +-
|
|
|
|
|
src/lib/helpers/youtubePlayerHandling.ts | 4 +-
|
|
|
|
|
src/lib/helpers/youtubePlayerReq.ts | 6 +-
|
|
|
|
|
src/lib/jobs/potoken.ts | 225 +++++++++++-----------
|
|
|
|
|
src/lib/jobs/worker.ts | 231 +++++++++++++++++++++++
|
|
|
|
|
src/lib/types/HonoVariables.ts | 4 +-
|
|
|
|
|
src/main.ts | 22 +--
|
|
|
|
|
7 files changed, 361 insertions(+), 133 deletions(-)
|
|
|
|
|
create mode 100644 src/lib/jobs/worker.ts
|
|
|
|
|
|
|
|
|
|
diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts
|
|
|
|
|
index 8cf34fd..2a3fdb3 100644
|
|
|
|
|
--- a/src/lib/helpers/config.ts
|
|
|
|
|
+++ b/src/lib/helpers/config.ts
|
|
|
|
|
@@ -1,7 +1,7 @@
|
|
|
|
|
import { z, ZodError } from "zod";
|
|
|
|
|
import { parse } from "@std/toml";
|
|
|
|
|
|
|
|
|
|
-const ConfigSchema = z.object({
|
|
|
|
|
+export const ConfigSchema = z.object({
|
|
|
|
|
server: z.object({
|
|
|
|
|
port: z.number().default(Number(Deno.env.get("PORT")) || 8282),
|
|
|
|
|
host: z.string().default(Deno.env.get("HOST") || "127.0.0.1"),
|
|
|
|
|
diff --git a/src/lib/helpers/youtubePlayerHandling.ts b/src/lib/helpers/youtubePlayerHandling.ts
|
|
|
|
|
index 7b5f0c1..62eb30c 100644
|
|
|
|
|
--- a/src/lib/helpers/youtubePlayerHandling.ts
|
|
|
|
|
+++ b/src/lib/helpers/youtubePlayerHandling.ts
|
|
|
|
|
@@ -1,8 +1,8 @@
|
|
|
|
|
import { ApiResponse, Innertube, YT } from "youtubei.js";
|
|
|
|
|
import { generateRandomString } from "youtubei.js/Utils";
|
|
|
|
|
import { compress, decompress } from "brotli";
|
|
|
|
|
-import type { BG } from "bgutils";
|
|
|
|
|
import { Metrics } from "../helpers/metrics.ts";
|
|
|
|
|
+import type { TokenMinter } from "../jobs/potoken.ts";
|
|
|
|
|
let youtubePlayerReqLocation = "youtubePlayerReq";
|
|
|
|
|
if (Deno.env.get("YT_PLAYER_REQ_LOCATION")) {
|
|
|
|
|
if (Deno.env.has("DENO_COMPILED")) {
|
|
|
|
|
@@ -31,7 +31,7 @@ export const youtubePlayerParsing = async ({
|
|
|
|
|
innertubeClient: Innertube;
|
|
|
|
|
videoId: string;
|
|
|
|
|
config: Config;
|
|
|
|
|
- tokenMinter: BG.WebPoMinter;
|
|
|
|
|
+ tokenMinter: TokenMinter;
|
|
|
|
|
metrics: Metrics | undefined;
|
|
|
|
|
overrideCache?: boolean;
|
|
|
|
|
}): Promise<object> => {
|
|
|
|
|
diff --git a/src/lib/helpers/youtubePlayerReq.ts b/src/lib/helpers/youtubePlayerReq.ts
|
|
|
|
|
index 884fae4..af78269 100644
|
|
|
|
|
--- a/src/lib/helpers/youtubePlayerReq.ts
|
|
|
|
|
+++ b/src/lib/helpers/youtubePlayerReq.ts
|
|
|
|
|
@@ -1,6 +1,6 @@
|
|
|
|
|
import { ApiResponse, Innertube } from "youtubei.js";
|
|
|
|
|
import NavigationEndpoint from "youtubei.js/NavigationEndpoint";
|
|
|
|
|
-import type { BG } from "bgutils";
|
|
|
|
|
+import type { TokenMinter } from "../jobs/potoken.ts";
|
|
|
|
|
|
|
|
|
|
import type { Config } from "./config.ts";
|
|
|
|
|
|
|
|
|
|
@@ -8,7 +8,7 @@ export const youtubePlayerReq = async (
|
|
|
|
|
innertubeClient: Innertube,
|
|
|
|
|
videoId: string,
|
|
|
|
|
config: Config,
|
|
|
|
|
- tokenMinter: BG.WebPoMinter,
|
|
|
|
|
+ tokenMinter: TokenMinter,
|
|
|
|
|
): Promise<ApiResponse> => {
|
|
|
|
|
const innertubeClientOauthEnabled = config.youtube_session.oauth_enabled;
|
|
|
|
|
|
|
|
|
|
@@ -21,7 +21,7 @@ export const youtubePlayerReq = async (
|
|
|
|
|
watchEndpoint: { videoId: videoId },
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
- const contentPoToken = await tokenMinter.mintAsWebsafeString(videoId);
|
|
|
|
|
+ const contentPoToken = await tokenMinter(videoId);
|
|
|
|
|
|
|
|
|
|
return watch_endpoint.call(innertubeClient.actions, {
|
|
|
|
|
playbackContext: {
|
|
|
|
|
diff --git a/src/lib/jobs/potoken.ts b/src/lib/jobs/potoken.ts
|
|
|
|
|
index e7da3c0..fe99e9c 100644
|
|
|
|
|
--- a/src/lib/jobs/potoken.ts
|
|
|
|
|
+++ b/src/lib/jobs/potoken.ts
|
|
|
|
|
@@ -1,6 +1,3 @@
|
|
|
|
|
-import { BG, buildURL, GOOG_API_KEY, USER_AGENT } from "bgutils";
|
|
|
|
|
-import type { WebPoSignalOutput } from "bgutils";
|
|
|
|
|
-import { JSDOM } from "jsdom";
|
|
|
|
|
import { Innertube, UniversalCache } from "youtubei.js";
|
|
|
|
|
import {
|
|
|
|
|
youtubePlayerParsing,
|
|
|
|
|
@@ -21,121 +18,134 @@ if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
|
|
|
|
|
}
|
|
|
|
|
const { getFetchClient } = await import(getFetchClientLocation);
|
|
|
|
|
|
|
|
|
|
-// Adapted from https://github.com/LuanRT/BgUtils/blob/main/examples/node/index.ts
|
|
|
|
|
-export const poTokenGenerate = async (
|
|
|
|
|
- innertubeClient: Innertube,
|
|
|
|
|
- config: Config,
|
|
|
|
|
- innertubeClientCache: UniversalCache,
|
|
|
|
|
- metrics: Metrics | undefined,
|
|
|
|
|
-): Promise<{ innertubeClient: Innertube; tokenMinter: BG.WebPoMinter }> => {
|
|
|
|
|
- if (innertubeClient.session.po_token) {
|
|
|
|
|
- innertubeClient = await Innertube.create({
|
|
|
|
|
- enable_session_cache: false,
|
|
|
|
|
- user_agent: USER_AGENT,
|
|
|
|
|
- retrieve_player: false,
|
|
|
|
|
- });
|
|
|
|
|
- }
|
|
|
|
|
+import { InputMessage, OutputMessageSchema } from "./worker.ts";
|
|
|
|
|
|
|
|
|
|
- const fetchImpl = await getFetchClient(config);
|
|
|
|
|
-
|
|
|
|
|
- const visitorData = innertubeClient.session.context.client.visitorData;
|
|
|
|
|
-
|
|
|
|
|
- if (!visitorData) {
|
|
|
|
|
- throw new Error("Could not get visitor data");
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- const dom = new JSDOM(
|
|
|
|
|
- '<!DOCTYPE html><html lang="en"><head><title></title></head><body></body></html>',
|
|
|
|
|
- {
|
|
|
|
|
- url: "https://www.youtube.com/",
|
|
|
|
|
- referrer: "https://www.youtube.com/",
|
|
|
|
|
- userAgent: USER_AGENT,
|
|
|
|
|
- },
|
|
|
|
|
- );
|
|
|
|
|
-
|
|
|
|
|
- Object.assign(globalThis, {
|
|
|
|
|
- window: dom.window,
|
|
|
|
|
- document: dom.window.document,
|
|
|
|
|
- location: dom.window.location,
|
|
|
|
|
- origin: dom.window.origin,
|
|
|
|
|
- });
|
|
|
|
|
+interface TokenGeneratorWorker extends Omit<Worker, "postMessage"> {
|
|
|
|
|
+ postMessage(message: InputMessage): void;
|
|
|
|
|
+}
|
|
|
|
|
|
|
|
|
|
- if (!Reflect.has(globalThis, "navigator")) {
|
|
|
|
|
- Object.defineProperty(globalThis, "navigator", {
|
|
|
|
|
- value: dom.window.navigator,
|
|
|
|
|
+const workers: TokenGeneratorWorker[] = [];
|
|
|
|
|
+
|
|
|
|
|
+function createMinter(worker: TokenGeneratorWorker) {
|
|
|
|
|
+ return (videoId: string): Promise<string> => {
|
|
|
|
|
+ const { promise, resolve } = Promise.withResolvers<string>();
|
|
|
|
|
+ // generate a UUID to identify the request as many minter calls
|
|
|
|
|
+ // may be made within a timespan, and this function will be
|
|
|
|
|
+ // informed about all of them until it's got its own
|
|
|
|
|
+ const requestId = crypto.randomUUID();
|
|
|
|
|
+ const listener = (message: MessageEvent) => {
|
|
|
|
|
+ const parsedMessage = OutputMessageSchema.parse(message.data);
|
|
|
|
|
+ if (
|
|
|
|
|
+ parsedMessage.type === "content-token" &&
|
|
|
|
|
+ parsedMessage.requestId === requestId
|
|
|
|
|
+ ) {
|
|
|
|
|
+ worker.removeEventListener("message", listener);
|
|
|
|
|
+ resolve(parsedMessage.contentToken);
|
|
|
|
|
+ }
|
|
|
|
|
+ };
|
|
|
|
|
+ worker.addEventListener("message", listener);
|
|
|
|
|
+ worker.postMessage({
|
|
|
|
|
+ type: "content-token-request",
|
|
|
|
|
+ videoId,
|
|
|
|
|
+ requestId,
|
|
|
|
|
});
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- const challengeResponse = await innertubeClient.getAttestationChallenge(
|
|
|
|
|
- "ENGAGEMENT_TYPE_UNBOUND",
|
|
|
|
|
- );
|
|
|
|
|
- if (!challengeResponse.bg_challenge) {
|
|
|
|
|
- throw new Error("Could not get challenge");
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- const interpreterUrl = challengeResponse.bg_challenge.interpreter_url
|
|
|
|
|
- .private_do_not_access_or_else_trusted_resource_url_wrapped_value;
|
|
|
|
|
- const bgScriptResponse = await fetchImpl(
|
|
|
|
|
- `http:${interpreterUrl}`,
|
|
|
|
|
- );
|
|
|
|
|
- const interpreterJavascript = await bgScriptResponse.text();
|
|
|
|
|
|
|
|
|
|
- if (interpreterJavascript) {
|
|
|
|
|
- new Function(interpreterJavascript)();
|
|
|
|
|
- } else throw new Error("Could not load VM");
|
|
|
|
|
+ return promise;
|
|
|
|
|
+ };
|
|
|
|
|
+}
|
|
|
|
|
|
|
|
|
|
- // Botguard currently surfaces a "Not implemented" error here, due to the environment
|
|
|
|
|
- // not having a valid Canvas API in JSDOM. At the time of writing, this doesn't cause
|
|
|
|
|
- // any issues as the Canvas check doesn't appear to be an enforced element of the checks
|
|
|
|
|
- console.log(
|
|
|
|
|
- '[INFO] the "Not implemented: HTMLCanvasElement.prototype.getContext" error is normal. Please do not open a bug report about it.',
|
|
|
|
|
- );
|
|
|
|
|
- const botguard = await BG.BotGuardClient.create({
|
|
|
|
|
- program: challengeResponse.bg_challenge.program,
|
|
|
|
|
- globalName: challengeResponse.bg_challenge.global_name,
|
|
|
|
|
- globalObj: globalThis,
|
|
|
|
|
- });
|
|
|
|
|
+export type TokenMinter = ReturnType<typeof createMinter>;
|
|
|
|
|
|
|
|
|
|
- const webPoSignalOutput: WebPoSignalOutput = [];
|
|
|
|
|
- const botguardResponse = await botguard.snapshot({ webPoSignalOutput });
|
|
|
|
|
- const requestKey = "O43z0dpjhgX20SCx4KAo";
|
|
|
|
|
+// Adapted from https://github.com/LuanRT/BgUtils/blob/main/examples/node/index.ts
|
|
|
|
|
+export const poTokenGenerate = (
|
|
|
|
|
+ config: Config,
|
|
|
|
|
+ innertubeClientCache: UniversalCache,
|
|
|
|
|
+ metrics: Metrics | undefined,
|
|
|
|
|
+): Promise<{ innertubeClient: Innertube; tokenMinter: TokenMinter }> => {
|
|
|
|
|
+ const { promise, resolve, reject } = Promise.withResolvers<
|
|
|
|
|
+ Awaited<ReturnType<typeof poTokenGenerate>>
|
|
|
|
|
+ >();
|
|
|
|
|
|
|
|
|
|
- const integrityTokenResponse = await fetchImpl(
|
|
|
|
|
- buildURL("GenerateIT", true),
|
|
|
|
|
+ const worker: TokenGeneratorWorker = new Worker(
|
|
|
|
|
+ new URL("./worker.ts", import.meta.url).href,
|
|
|
|
|
{
|
|
|
|
|
- method: "POST",
|
|
|
|
|
- headers: {
|
|
|
|
|
- "content-type": "application/json+protobuf",
|
|
|
|
|
- "x-goog-api-key": GOOG_API_KEY,
|
|
|
|
|
- "x-user-agent": "grpc-web-javascript/0.1",
|
|
|
|
|
- "user-agent": USER_AGENT,
|
|
|
|
|
- },
|
|
|
|
|
- body: JSON.stringify([requestKey, botguardResponse]),
|
|
|
|
|
+ type: "module",
|
|
|
|
|
+ name: "PO Token Generator",
|
|
|
|
|
},
|
|
|
|
|
);
|
|
|
|
|
+ // take note of the worker so we can kill it once a new one takes its place
|
|
|
|
|
+ workers.push(worker);
|
|
|
|
|
+ worker.addEventListener("message", async (event) => {
|
|
|
|
|
+ const parsedMessage = OutputMessageSchema.parse(event.data);
|
|
|
|
|
+
|
|
|
|
|
+ // worker is listening for messages
|
|
|
|
|
+ if (parsedMessage.type === "ready") {
|
|
|
|
|
+ const untypedPostMessage = worker.postMessage.bind(worker);
|
|
|
|
|
+ worker.postMessage = (message: InputMessage) =>
|
|
|
|
|
+ untypedPostMessage(message);
|
|
|
|
|
+ worker.postMessage({ type: "initialise", config });
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
|
|
- const response = await integrityTokenResponse.json() as unknown[];
|
|
|
|
|
-
|
|
|
|
|
- if (typeof response[0] !== "string") {
|
|
|
|
|
- throw new Error("Could not get integrity token");
|
|
|
|
|
- }
|
|
|
|
|
+ if (parsedMessage.type === "error") {
|
|
|
|
|
+ console.log({ errorFromWorker: parsedMessage.error });
|
|
|
|
|
+ worker.terminate();
|
|
|
|
|
+ reject(parsedMessage.error);
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
|
|
- const integrityTokenBasedMinter = await BG.WebPoMinter.create({
|
|
|
|
|
- integrityToken: response[0],
|
|
|
|
|
- }, webPoSignalOutput);
|
|
|
|
|
+ // worker is initialised and has passed back a session token and visitor data
|
|
|
|
|
+ if (parsedMessage.type === "initialised") {
|
|
|
|
|
+ try {
|
|
|
|
|
+ const instantiatedInnertubeClient = await Innertube.create({
|
|
|
|
|
+ enable_session_cache: false,
|
|
|
|
|
+ po_token: parsedMessage.sessionPoToken,
|
|
|
|
|
+ visitor_data: parsedMessage.visitorData,
|
|
|
|
|
+ fetch: getFetchClient(config),
|
|
|
|
|
+ cache: innertubeClientCache,
|
|
|
|
|
+ generate_session_locally: true,
|
|
|
|
|
+ });
|
|
|
|
|
+ const minter = createMinter(worker);
|
|
|
|
|
+ // check token from minter
|
|
|
|
|
+ await checkToken({
|
|
|
|
|
+ instantiatedInnertubeClient,
|
|
|
|
|
+ config,
|
|
|
|
|
+ integrityTokenBasedMinter: minter,
|
|
|
|
|
+ metrics,
|
|
|
|
|
+ });
|
|
|
|
|
+ console.log("Successfully generated PO token");
|
|
|
|
|
+ const numberToKill = workers.length - 1;
|
|
|
|
|
+ for (let i = 0; i < numberToKill; i++) {
|
|
|
|
|
+ const workerToKill = workers.shift();
|
|
|
|
|
+ workerToKill?.terminate();
|
|
|
|
|
+ }
|
|
|
|
|
+ return resolve({
|
|
|
|
|
+ innertubeClient: instantiatedInnertubeClient,
|
|
|
|
|
+ tokenMinter: minter,
|
|
|
|
|
+ });
|
|
|
|
|
+ } catch (err) {
|
|
|
|
|
+ console.log("Failed to get valid PO token, will retry", {
|
|
|
|
|
+ err,
|
|
|
|
|
+ });
|
|
|
|
|
+ worker.terminate();
|
|
|
|
|
+ reject(err);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ });
|
|
|
|
|
|
|
|
|
|
- const sessionPoToken = await integrityTokenBasedMinter.mintAsWebsafeString(
|
|
|
|
|
- visitorData,
|
|
|
|
|
- );
|
|
|
|
|
+ return promise;
|
|
|
|
|
+};
|
|
|
|
|
|
|
|
|
|
- const instantiatedInnertubeClient = await Innertube.create({
|
|
|
|
|
- enable_session_cache: false,
|
|
|
|
|
- po_token: sessionPoToken,
|
|
|
|
|
- visitor_data: visitorData,
|
|
|
|
|
- fetch: getFetchClient(config),
|
|
|
|
|
- cache: innertubeClientCache,
|
|
|
|
|
- generate_session_locally: true,
|
|
|
|
|
- });
|
|
|
|
|
+async function checkToken({
|
|
|
|
|
+ instantiatedInnertubeClient,
|
|
|
|
|
+ config,
|
|
|
|
|
+ integrityTokenBasedMinter,
|
|
|
|
|
+ metrics,
|
|
|
|
|
+}: {
|
|
|
|
|
+ instantiatedInnertubeClient: Innertube;
|
|
|
|
|
+ config: Config;
|
|
|
|
|
+ integrityTokenBasedMinter: TokenMinter;
|
|
|
|
|
+ metrics: Metrics | undefined,
|
|
|
|
|
+}) {
|
|
|
|
|
+ const fetchImpl = getFetchClient(config);
|
|
|
|
|
|
|
|
|
|
if (config.jobs.youtube_session.po_token_check) {
|
|
|
|
|
try {
|
|
|
|
|
@@ -183,9 +193,4 @@ export const poTokenGenerate = async (
|
|
|
|
|
throw err;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
-
|
|
|
|
|
- return {
|
|
|
|
|
- innertubeClient: instantiatedInnertubeClient,
|
|
|
|
|
- tokenMinter: integrityTokenBasedMinter,
|
|
|
|
|
- };
|
|
|
|
|
-};
|
|
|
|
|
+}
|
|
|
|
|
diff --git a/src/lib/jobs/worker.ts b/src/lib/jobs/worker.ts
|
|
|
|
|
new file mode 100644
|
|
|
|
|
index 0000000..0d62bfc
|
|
|
|
|
--- /dev/null
|
|
|
|
|
+++ b/src/lib/jobs/worker.ts
|
|
|
|
|
@@ -0,0 +1,231 @@
|
|
|
|
|
+/// <reference lib="webworker" />
|
|
|
|
|
+
|
|
|
|
|
+import { z } from "zod";
|
|
|
|
|
+import { Config, ConfigSchema } from "../helpers/config.ts";
|
|
|
|
|
+import { BG, buildURL, GOOG_API_KEY, USER_AGENT } from "bgutils";
|
|
|
|
|
+import type { WebPoSignalOutput } from "bgutils";
|
|
|
|
|
+import { JSDOM } from "jsdom";
|
|
|
|
|
+import { Innertube } from "youtubei.js";
|
|
|
|
|
+let getFetchClientLocation = "getFetchClient";
|
|
|
|
|
+if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
|
|
|
|
|
+ if (Deno.env.has("DENO_COMPILED")) {
|
|
|
|
|
+ getFetchClientLocation = Deno.mainModule.replace("src/main.ts", "") +
|
|
|
|
|
+ Deno.env.get("GET_FETCH_CLIENT_LOCATION");
|
|
|
|
|
+ } else {
|
|
|
|
|
+ getFetchClientLocation = Deno.env.get(
|
|
|
|
|
+ "GET_FETCH_CLIENT_LOCATION",
|
|
|
|
|
+ ) as string;
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+type FetchFunction = typeof fetch;
|
|
|
|
|
+const { getFetchClient }: {
|
|
|
|
|
+ getFetchClient: (config: Config) => Promise<FetchFunction>;
|
|
|
|
|
+} = await import(getFetchClientLocation);
|
|
|
|
|
+
|
|
|
|
|
+// ---- Messages to send to the webworker ----
|
|
|
|
|
+const InputInitialiseSchema = z.object({
|
|
|
|
|
+ type: z.literal("initialise"),
|
|
|
|
|
+ config: ConfigSchema,
|
|
|
|
|
+}).strict();
|
|
|
|
|
+
|
|
|
|
|
+const InputContentTokenSchema = z.object({
|
|
|
|
|
+ type: z.literal("content-token-request"),
|
|
|
|
|
+ videoId: z.string(),
|
|
|
|
|
+ requestId: z.string().uuid(),
|
|
|
|
|
+}).strict();
|
|
|
|
|
+export type InputInitialise = z.infer<typeof InputInitialiseSchema>;
|
|
|
|
|
+export type InputContentToken = z.infer<typeof InputContentTokenSchema>;
|
|
|
|
|
+const InputMessageSchema = z.union([
|
|
|
|
|
+ InputInitialiseSchema,
|
|
|
|
|
+ InputContentTokenSchema,
|
|
|
|
|
+]);
|
|
|
|
|
+export type InputMessage = z.infer<typeof InputMessageSchema>;
|
|
|
|
|
+
|
|
|
|
|
+// ---- Messages that the webworker sends to the parent ----
|
|
|
|
|
+const OutputReadySchema = z.object({
|
|
|
|
|
+ type: z.literal("ready"),
|
|
|
|
|
+}).strict();
|
|
|
|
|
+
|
|
|
|
|
+const OutputInitialiseSchema = z.object({
|
|
|
|
|
+ type: z.literal("initialised"),
|
|
|
|
|
+ sessionPoToken: z.string(),
|
|
|
|
|
+ visitorData: z.string(),
|
|
|
|
|
+}).strict();
|
|
|
|
|
+
|
|
|
|
|
+const OutputContentTokenSchema = z.object({
|
|
|
|
|
+ type: z.literal("content-token"),
|
|
|
|
|
+ contentToken: z.string(),
|
|
|
|
|
+ requestId: InputContentTokenSchema.shape.requestId,
|
|
|
|
|
+}).strict();
|
|
|
|
|
+
|
|
|
|
|
+const OutputErrorSchema = z.object({
|
|
|
|
|
+ type: z.literal("error"),
|
|
|
|
|
+ error: z.any(),
|
|
|
|
|
+}).strict();
|
|
|
|
|
+export const OutputMessageSchema = z.union([
|
|
|
|
|
+ OutputReadySchema,
|
|
|
|
|
+ OutputInitialiseSchema,
|
|
|
|
|
+ OutputContentTokenSchema,
|
|
|
|
|
+ OutputErrorSchema,
|
|
|
|
|
+]);
|
|
|
|
|
+type OutputMessage = z.infer<typeof OutputMessageSchema>;
|
|
|
|
|
+
|
|
|
|
|
+const IntegrityTokenResponse = z.tuple([z.string()]).rest(z.any());
|
|
|
|
|
+
|
|
|
|
|
+const isWorker = typeof WorkerGlobalScope !== "undefined" &&
|
|
|
|
|
+ self instanceof WorkerGlobalScope;
|
|
|
|
|
+if (isWorker) {
|
|
|
|
|
+ // helper function to force type-checking
|
|
|
|
|
+ const untypedPostmessage = self.postMessage.bind(self);
|
|
|
|
|
+ const postMessage = (message: OutputMessage) => {
|
|
|
|
|
+ untypedPostmessage(message);
|
|
|
|
|
+ };
|
|
|
|
|
+
|
|
|
|
|
+ let minter: BG.WebPoMinter;
|
|
|
|
|
+
|
|
|
|
|
+ onmessage = async (event) => {
|
|
|
|
|
+ const message = InputMessageSchema.parse(event.data);
|
|
|
|
|
+ if (message.type === "initialise") {
|
|
|
|
|
+ const fetchImpl: typeof fetch = await getFetchClient(
|
|
|
|
|
+ message.config,
|
|
|
|
|
+ );
|
|
|
|
|
+ try {
|
|
|
|
|
+ const {
|
|
|
|
|
+ sessionPoToken,
|
|
|
|
|
+ visitorData,
|
|
|
|
|
+ generatedMinter,
|
|
|
|
|
+ } = await setup({ fetchImpl });
|
|
|
|
|
+ minter = generatedMinter;
|
|
|
|
|
+ postMessage({
|
|
|
|
|
+ type: "initialised",
|
|
|
|
|
+ sessionPoToken,
|
|
|
|
|
+ visitorData,
|
|
|
|
|
+ });
|
|
|
|
|
+ } catch (err) {
|
|
|
|
|
+ postMessage({ type: "error", error: err });
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ // this is called every time a video needs a content token
|
|
|
|
|
+ if (message.type === "content-token-request") {
|
|
|
|
|
+ if (!minter) {
|
|
|
|
|
+ throw new Error(
|
|
|
|
|
+ "Minter not yet ready, must initialise first",
|
|
|
|
|
+ );
|
|
|
|
|
+ }
|
|
|
|
|
+ const contentToken = await minter.mintAsWebsafeString(
|
|
|
|
|
+ message.videoId,
|
|
|
|
|
+ );
|
|
|
|
|
+ postMessage({
|
|
|
|
|
+ type: "content-token",
|
|
|
|
|
+ contentToken,
|
|
|
|
|
+ requestId: message.requestId,
|
|
|
|
|
+ });
|
|
|
|
|
+ }
|
|
|
|
|
+ };
|
|
|
|
|
+
|
|
|
|
|
+ postMessage({ type: "ready" });
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+async function setup(
|
|
|
|
|
+ { fetchImpl }: { fetchImpl: FetchFunction },
|
|
|
|
|
+) {
|
|
|
|
|
+ const innertubeClient = await Innertube.create({
|
|
|
|
|
+ enable_session_cache: false,
|
|
|
|
|
+ user_agent: USER_AGENT,
|
|
|
|
|
+ retrieve_player: false,
|
|
|
|
|
+ });
|
|
|
|
|
+
|
|
|
|
|
+ const visitorData = innertubeClient.session.context.client.visitorData;
|
|
|
|
|
+
|
|
|
|
|
+ if (!visitorData) {
|
|
|
|
|
+ throw new Error("Could not get visitor data");
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const dom = new JSDOM(
|
|
|
|
|
+ '<!DOCTYPE html><html lang="en"><head><title></title></head><body></body></html>',
|
|
|
|
|
+ {
|
|
|
|
|
+ url: "https://www.youtube.com/",
|
|
|
|
|
+ referrer: "https://www.youtube.com/",
|
|
|
|
|
+ userAgent: USER_AGENT,
|
|
|
|
|
+ },
|
|
|
|
|
+ );
|
|
|
|
|
+
|
|
|
|
|
+ Object.assign(globalThis, {
|
|
|
|
|
+ window: dom.window,
|
|
|
|
|
+ document: dom.window.document,
|
|
|
|
|
+ // location: dom.window.location, // --- doesn't seem to be necessary and the Web Worker doesn't like it
|
|
|
|
|
+ origin: dom.window.origin,
|
|
|
|
|
+ });
|
|
|
|
|
+
|
|
|
|
|
+ if (!Reflect.has(globalThis, "navigator")) {
|
|
|
|
|
+ Object.defineProperty(globalThis, "navigator", {
|
|
|
|
|
+ value: dom.window.navigator,
|
|
|
|
|
+ });
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const challengeResponse = await innertubeClient.getAttestationChallenge(
|
|
|
|
|
+ "ENGAGEMENT_TYPE_UNBOUND",
|
|
|
|
|
+ );
|
|
|
|
|
+ if (!challengeResponse.bg_challenge) {
|
|
|
|
|
+ throw new Error("Could not get challenge");
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const interpreterUrl = challengeResponse.bg_challenge.interpreter_url
|
|
|
|
|
+ .private_do_not_access_or_else_trusted_resource_url_wrapped_value;
|
|
|
|
|
+ const bgScriptResponse = await fetchImpl(
|
|
|
|
|
+ `https:${interpreterUrl}`,
|
|
|
|
|
+ );
|
|
|
|
|
+ const interpreterJavascript = await bgScriptResponse.text();
|
|
|
|
|
+
|
|
|
|
|
+ if (interpreterJavascript) {
|
|
|
|
|
+ new Function(interpreterJavascript)();
|
|
|
|
|
+ } else throw new Error("Could not load VM");
|
|
|
|
|
+
|
|
|
|
|
+ // Botguard currently surfaces a "Not implemented" error here, due to the environment
|
|
|
|
|
+ // not having a valid Canvas API in JSDOM. At the time of writing, this doesn't cause
|
|
|
|
|
+ // any issues as the Canvas check doesn't appear to be an enforced element of the checks
|
|
|
|
|
+ console.log(
|
|
|
|
|
+ '[INFO] the "Not implemented: HTMLCanvasElement.prototype.getContext" error is normal. Please do not open a bug report about it.',
|
|
|
|
|
+ );
|
|
|
|
|
+ const botguard = await BG.BotGuardClient.create({
|
|
|
|
|
+ program: challengeResponse.bg_challenge.program,
|
|
|
|
|
+ globalName: challengeResponse.bg_challenge.global_name,
|
|
|
|
|
+ globalObj: globalThis,
|
|
|
|
|
+ });
|
|
|
|
|
+
|
|
|
|
|
+ const webPoSignalOutput: WebPoSignalOutput = [];
|
|
|
|
|
+ const botguardResponse = await botguard.snapshot({ webPoSignalOutput });
|
|
|
|
|
+ const requestKey = "O43z0dpjhgX20SCx4KAo";
|
|
|
|
|
+
|
|
|
|
|
+ const integrityTokenResponse = await fetchImpl(
|
|
|
|
|
+ buildURL("GenerateIT", true),
|
|
|
|
|
+ {
|
|
|
|
|
+ method: "POST",
|
|
|
|
|
+ headers: {
|
|
|
|
|
+ "content-type": "application/json+protobuf",
|
|
|
|
|
+ "x-goog-api-key": GOOG_API_KEY,
|
|
|
|
|
+ "x-user-agent": "grpc-web-javascript/0.1",
|
|
|
|
|
+ "user-agent": USER_AGENT,
|
|
|
|
|
+ },
|
|
|
|
|
+ body: JSON.stringify([requestKey, botguardResponse]),
|
|
|
|
|
+ },
|
|
|
|
|
+ );
|
|
|
|
|
+ const integrityTokenBody = IntegrityTokenResponse.parse(
|
|
|
|
|
+ await integrityTokenResponse.json(),
|
|
|
|
|
+ );
|
|
|
|
|
+
|
|
|
|
|
+ const integrityTokenBasedMinter = await BG.WebPoMinter.create({
|
|
|
|
|
+ integrityToken: integrityTokenBody[0],
|
|
|
|
|
+ }, webPoSignalOutput);
|
|
|
|
|
+
|
|
|
|
|
+ const sessionPoToken = await integrityTokenBasedMinter.mintAsWebsafeString(
|
|
|
|
|
+ visitorData,
|
|
|
|
|
+ );
|
|
|
|
|
+
|
|
|
|
|
+ return {
|
|
|
|
|
+ sessionPoToken,
|
|
|
|
|
+ visitorData,
|
|
|
|
|
+ generatedMinter: integrityTokenBasedMinter,
|
|
|
|
|
+ };
|
|
|
|
|
+}
|
|
|
|
|
diff --git a/src/lib/types/HonoVariables.ts b/src/lib/types/HonoVariables.ts
|
|
|
|
|
index 1e9b7a2..ab385ab 100644
|
|
|
|
|
--- a/src/lib/types/HonoVariables.ts
|
|
|
|
|
+++ b/src/lib/types/HonoVariables.ts
|
|
|
|
|
@@ -1,11 +1,11 @@
|
|
|
|
|
import { Innertube } from "youtubei.js";
|
|
|
|
|
-import { BG } from "bgutils";
|
|
|
|
|
+import type { TokenMinter } from "../jobs/potoken.ts";
|
|
|
|
|
import type { Config } from "../helpers/config.ts";
|
|
|
|
|
import { Metrics } from "../helpers/metrics.ts";
|
|
|
|
|
|
|
|
|
|
export type HonoVariables = {
|
|
|
|
|
innertubeClient: Innertube;
|
|
|
|
|
config: Config;
|
|
|
|
|
- tokenMinter: BG.WebPoMinter;
|
|
|
|
|
+ tokenMinter: TokenMinter;
|
|
|
|
|
metrics: Metrics | undefined;
|
|
|
|
|
};
|
|
|
|
|
diff --git a/src/main.ts b/src/main.ts
|
|
|
|
|
index 851cd71..cc9e97d 100644
|
|
|
|
|
--- a/src/main.ts
|
|
|
|
|
+++ b/src/main.ts
|
|
|
|
|
@@ -1,10 +1,9 @@
|
|
|
|
|
import { Hono } from "hono";
|
|
|
|
|
import { routes } from "./routes/index.ts";
|
|
|
|
|
import { Innertube, UniversalCache } from "youtubei.js";
|
|
|
|
|
-import { poTokenGenerate } from "./lib/jobs/potoken.ts";
|
|
|
|
|
+import { poTokenGenerate, type TokenMinter } from "./lib/jobs/potoken.ts";
|
|
|
|
|
import { USER_AGENT } from "bgutils";
|
|
|
|
|
import { retry } from "@std/async";
|
|
|
|
|
-import type { BG } from "bgutils";
|
|
|
|
|
import type { HonoVariables } from "./lib/types/HonoVariables.ts";
|
|
|
|
|
|
|
|
|
|
import { parseConfig } from "./lib/helpers/config.ts";
|
|
|
|
|
@@ -30,7 +29,7 @@ declare module "hono" {
|
|
|
|
|
const app = new Hono();
|
|
|
|
|
const metrics = config.server.enable_metrics ? new Metrics() : undefined;
|
|
|
|
|
|
|
|
|
|
-let tokenMinter: BG.WebPoMinter;
|
|
|
|
|
+let tokenMinter: TokenMinter;
|
|
|
|
|
let innertubeClient: Innertube;
|
|
|
|
|
let innertubeClientFetchPlayer = true;
|
|
|
|
|
const innertubeClientOauthEnabled = config.youtube_session.oauth_enabled;
|
|
|
|
|
@@ -65,7 +64,6 @@ if (!innertubeClientOauthEnabled) {
|
|
|
|
|
({ innertubeClient, tokenMinter } = await retry(
|
|
|
|
|
poTokenGenerate.bind(
|
|
|
|
|
poTokenGenerate,
|
|
|
|
|
- innertubeClient,
|
|
|
|
|
config,
|
|
|
|
|
innertubeClientCache,
|
|
|
|
|
metrics,
|
|
|
|
|
@@ -79,17 +77,11 @@ if (!innertubeClientOauthEnabled) {
|
|
|
|
|
{ backoffSchedule: [5_000, 15_000, 60_000, 180_000] },
|
|
|
|
|
async () => {
|
|
|
|
|
if (innertubeClientJobPoTokenEnabled) {
|
|
|
|
|
- try {
|
|
|
|
|
- ({ innertubeClient, tokenMinter } = await poTokenGenerate(
|
|
|
|
|
- innertubeClient,
|
|
|
|
|
- config,
|
|
|
|
|
- innertubeClientCache,
|
|
|
|
|
- metrics,
|
|
|
|
|
- ));
|
|
|
|
|
- } catch (err) {
|
|
|
|
|
- metrics?.potokenGenerationFailure.inc();
|
|
|
|
|
- throw err;
|
|
|
|
|
- }
|
|
|
|
|
+ ({ innertubeClient, tokenMinter } = await poTokenGenerate(
|
|
|
|
|
+ config,
|
|
|
|
|
+ innertubeClientCache,
|
|
|
|
|
+ metrics,
|
|
|
|
|
+ ));
|
|
|
|
|
} else {
|
|
|
|
|
innertubeClient = await Innertube.create({
|
|
|
|
|
enable_session_cache: false,
|
|
|
|
|
--
|
|
|
|
|
2.49.0
|
|
|
|
|
|
