Fijxu/invidious-companion-patches
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

update patches and invidious-companion
Some checks failed
Build and Push Docker Image / build-and-push (push) Failing after 11s
Details

Browse source
This commit is contained in:
Fijxu 2025-04-05 03:33:59 -03:00
parent d8259f12ff
commit b5ccecf5c6
Signed by: Fijxu
GPG key ID: 32C1DDF333EDA6A4
14 changed files with 68 additions and 3437 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
invidious-companion

@ -1 +1 @@
Subproject commit ce3ba082d2d04fb85c74004835287f56faccb41a
Subproject commit 0f41fe19781d9ce29b2043fbc8b184197ba2b6bf

4
patches/0001-ci-update-deno-to-2.2.5.patch
View file

@ -1,7 +1,7 @@
From ad4b5aca25433218a7e903acffece4ebf222127e Mon Sep 17 00:00:00 2001
From 070f466db16386cf6e54b7880d84f79b4413cba4 Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Mon, 24 Mar 2025 19:37:34 -0300
Subject: [PATCH 01/13] ci: update deno to 2.2.5
Subject: [PATCH 01/11] ci: update deno to 2.2.5
---
Dockerfile | 2 +-

20
patches/0002-feat-add-support-for-an-external-videoplayback-proxy.patch
View file

@ -1,7 +1,7 @@
From 38a460ca36d462c7d4396a4a42266c318d56c505 Mon Sep 17 00:00:00 2001
From b652b922781ddbd3889a03a101015edee5cfa06f Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Mon, 24 Mar 2025 18:44:10 -0300
Subject: [PATCH 02/13] feat: add support for an external videoplayback proxy
Subject: [PATCH 02/11] feat: add support for an external videoplayback proxy
---
config/config.example.toml | 1 +
@ -11,10 +11,10 @@ Subject: [PATCH 02/13] feat: add support for an external videoplayback proxy
4 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/config/config.example.toml b/config/config.example.toml
index 5989656..0b4d7b8 100644
index cce4704..4aec8e9 100644
--- a/config/config.example.toml
+++ b/config/config.example.toml
@@ -27,6 +27,7 @@
@@ -28,6 +28,7 @@
# #proxy = "" # env variable: PROXY
# # Enable YouTube new video format UMP
# ump = false
@ -23,10 +23,10 @@ index 5989656..0b4d7b8 100644
###
# Network call timeouts when talking to YouTube.
diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts
index 426426f..273fe95 100644
index cd851d7..a233f83 100644
--- a/src/lib/helpers/config.ts
+++ b/src/lib/helpers/config.ts
@@ -29,6 +29,9 @@ const ConfigSchema = z.object({
@@ -32,6 +32,9 @@ export const ConfigSchema = z.object({
debounce_multiplier: z.number().optional(),
}).strict().optional(),
}).strict().optional(),
@ -37,10 +37,10 @@ index 426426f..273fe95 100644
jobs: z.object({
youtube_session: z.object({
diff --git a/src/routes/invidious_routes/dashManifest.ts b/src/routes/invidious_routes/dashManifest.ts
index 3834601..d69f1cf 100644
index d241407..b4446b6 100644
--- a/src/routes/invidious_routes/dashManifest.ts
+++ b/src/routes/invidious_routes/dashManifest.ts
@@ -91,8 +91,10 @@ dashManifest.get("/:videoId", async (c) => {
@@ -93,8 +93,10 @@ dashManifest.get("/:videoId", async (c) => {
queryParams.set("enc", "true");
queryParams.set("data", encryptedParams);
}
@ -54,10 +54,10 @@ index 3834601..d69f1cf 100644
} else {
return dashUrl;
diff --git a/src/routes/invidious_routes/latestVersion.ts b/src/routes/invidious_routes/latestVersion.ts
index 6421904..07b070a 100644
index 331be18..74dd090 100644
--- a/src/routes/invidious_routes/latestVersion.ts
+++ b/src/routes/invidious_routes/latestVersion.ts
@@ -83,7 +83,8 @@ latestVersion.get("/", async (c) => {
@@ -85,7 +85,8 @@ latestVersion.get("/", async (c) => {
queryParams.set("enc", "true");
queryParams.set("data", encryptedParams);
}

18
patches/0003-feat-report-the-external-videoplayback-proxy-via-inf.patch
View file

@ -1,31 +1,35 @@
From ca369efa1837eff09e782f2e2d363ee23d66a43e Mon Sep 17 00:00:00 2001
From 8939348608c58d7e658ec4deab390e7e5acf3718 Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Mon, 24 Mar 2025 18:52:53 -0300
Subject: [PATCH 03/13] feat: report the external videoplayback proxy via /info
Subject: [PATCH 03/11] feat: report the external videoplayback proxy via /info
endpoint
---
src/routes/index.ts | 2 ++
src/routes/index.ts | 6 ++----
src/routes/info.ts | 17 +++++++++++++++++
2 files changed, 19 insertions(+)
2 files changed, 19 insertions(+), 4 deletions(-)
create mode 100644 src/routes/info.ts
diff --git a/src/routes/index.ts b/src/routes/index.ts
index e67b618..6448e3d 100644
index 5aa9fa1..6448e3d 100644
--- a/src/routes/index.ts
+++ b/src/routes/index.ts
@@ -10,6 +10,7 @@ import getDownloadHandler from "./invidious_routes/download.ts";
@@ -10,7 +10,7 @@ import getDownloadHandler from "./invidious_routes/download.ts";
import videoPlaybackProxy from "./videoPlaybackProxy.ts";
import health from "./health.ts";
import type { Config } from "../lib/helpers/config.ts";
-import metrics from "./metrics.ts";
+import info from "./info.ts";
export const routes = (
app: Hono,
@@ -32,4 +33,5 @@ export const routes = (
@@ -33,7 +33,5 @@ export const routes = (
app.route("/api/v1/captions", invidiousCaptionsApi);
app.route("/videoplayback", videoPlaybackProxy);
app.route("/healthz", health);
- if (config.server.enable_metrics) {
- app.route("/metrics", metrics);
- }
+ app.route("/info", info);
};
diff --git a/src/routes/info.ts b/src/routes/info.ts

22
patches/0004-feat-add-resolution-limit-on-DASH-streams-to-save-ba.patch
View file

@ -1,7 +1,7 @@
From cd276db4eee8885f0022dc5a72ea069de8b54fd1 Mon Sep 17 00:00:00 2001
From ee6196d73a27ef3155189583c4a0ae4aad566a82 Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Mon, 24 Mar 2025 19:02:01 -0300
Subject: [PATCH 04/13] feat: add resolution limit on DASH streams to save
Subject: [PATCH 04/11] feat: add resolution limit on DASH streams to save
bandwidth
---
@ -11,24 +11,24 @@ Subject: [PATCH 04/13] feat: add resolution limit on DASH streams to save
3 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/config/config.example.toml b/config/config.example.toml
index 0b4d7b8..6d342bb 100644
index 4aec8e9..8e2af1c 100644
--- a/config/config.example.toml
+++ b/config/config.example.toml
@@ -16,6 +16,7 @@
# secret_key = "CHANGE_ME" # env variable: SERVER_SECRET_KEY
@@ -17,6 +17,7 @@
# verify_requests = false
# encrypt_query_params = false # env variable: SERVER_ENCRYPT_QUERY_PARAMS
# enable_metrics = false # env variable: SERVER_ENABLE_METRICS
+# max_dash_resolution = 1080
# [cache]
# enabled = true
diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts
index 273fe95..d1968fe 100644
index a233f83..e559271 100644
--- a/src/lib/helpers/config.ts
+++ b/src/lib/helpers/config.ts
@@ -12,6 +12,9 @@ const ConfigSchema = z.object({
encrypt_query_params: z.boolean().default(
Deno.env.get("SERVER_ENCRYPT_QUERY_PARAMS") === "true" || false,
@@ -15,6 +15,9 @@ export const ConfigSchema = z.object({
enable_metrics: z.boolean().default(
Deno.env.get("SERVER_ENABLE_METRICS") === "true" || false,
),
+ max_dash_resolution: z.number().default(
+ Number(Deno.env.get("SERVER_MAX_DASH_RESOLUTION")),
@ -37,10 +37,10 @@ index 273fe95..d1968fe 100644
cache: z.object({
enabled: z.boolean().default(true),
diff --git a/src/routes/invidious_routes/dashManifest.ts b/src/routes/invidious_routes/dashManifest.ts
index d69f1cf..10b23d8 100644
index b4446b6..a691d6e 100644
--- a/src/routes/invidious_routes/dashManifest.ts
+++ b/src/routes/invidious_routes/dashManifest.ts
@@ -53,7 +53,8 @@ dashManifest.get("/:videoId", async (c) => {
@@ -55,7 +55,8 @@ dashManifest.get("/:videoId", async (c) => {
videoInfo.streaming_data.adaptive_formats = videoInfo
.streaming_data.adaptive_formats
.filter((i) =>

8
patches/0005-feat-add-env-variable-to-set-verify_requests.patch
View file

@ -1,17 +1,17 @@
From 15a903e7478366173e4ac90f1eb96526fc94df8d Mon Sep 17 00:00:00 2001
From fb707c08fae6bad7477ffae8570ea1746243b7a2 Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Mon, 24 Mar 2025 19:06:04 -0300
Subject: [PATCH 05/13] feat: add env variable to set verify_requests
Subject: [PATCH 05/11] feat: add env variable to set verify_requests
---
src/lib/helpers/config.ts | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts
index d1968fe..a5221c9 100644
index e559271..cd0489d 100644
--- a/src/lib/helpers/config.ts
+++ b/src/lib/helpers/config.ts
@@ -8,7 +8,9 @@ const ConfigSchema = z.object({
@@ -8,7 +8,9 @@ export const ConfigSchema = z.object({
secret_key: z.string().length(16).default(
Deno.env.get("SERVER_SECRET_KEY") || "",
),

4
patches/0006-feat-add-support-for-multiple-proxies.patch
View file

@ -1,7 +1,7 @@
From d2fd6b4cc2aa123b82ba0c11a5d4593ca8cdc4dc Mon Sep 17 00:00:00 2001
From a6742b9291518bd3dbe2599ec9acab30cfbf5e50 Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Mon, 24 Mar 2025 19:20:52 -0300
Subject: [PATCH 06/13] feat: add support for multiple proxies
Subject: [PATCH 06/11] feat: add support for multiple proxies
---
src/lib/helpers/getFetchClient.ts | 17 ++++++++++++++++-

16
patches/0007-feat-add-option-to-disable-potoken-generation-check.patch
View file

@ -1,7 +1,7 @@
From 600734d4b7731ad70605770cf99060d5af382647 Mon Sep 17 00:00:00 2001
From ef1188070f4beaab5a7332ea1542a06d47099a82 Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Mon, 24 Mar 2025 20:34:33 -0300
Subject: [PATCH 07/13] feat: add option to disable potoken generation check
Subject: [PATCH 07/11] feat: add option to disable potoken generation check
---
config/config.example.toml | 1 +
@ -10,10 +10,10 @@ Subject: [PATCH 07/13] feat: add option to disable potoken generation check
3 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/config/config.example.toml b/config/config.example.toml
index 6d342bb..969c768 100644
index 8e2af1c..b1cb4ee 100644
--- a/config/config.example.toml
+++ b/config/config.example.toml
@@ -52,6 +52,7 @@
@@ -53,6 +53,7 @@
# [jobs.youtube_session]
# po_token_enabled = true # whether to generate PO tokens
# frequency = "*/5 * * * *" # frequency of PO token refresh in cron format
@ -22,10 +22,10 @@ index 6d342bb..969c768 100644
# [youtube_session]
# oauth_enabled = false
diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts
index a5221c9..e6b1070 100644
index cd0489d..6004753 100644
--- a/src/lib/helpers/config.ts
+++ b/src/lib/helpers/config.ts
@@ -42,6 +42,15 @@ const ConfigSchema = z.object({
@@ -45,6 +45,15 @@ export const ConfigSchema = z.object({
youtube_session: z.object({
po_token_enabled: z.boolean().default(true),
frequency: z.string().default("*/5 * * * *"),
@ -42,10 +42,10 @@ index a5221c9..e6b1070 100644
}).strict().default({}),
youtube_session: z.object({
diff --git a/src/lib/jobs/potoken.ts b/src/lib/jobs/potoken.ts
index 6867082..3e5dfc0 100644
index a628996..8ec2034 100644
--- a/src/lib/jobs/potoken.ts
+++ b/src/lib/jobs/potoken.ts
@@ -166,7 +166,9 @@ export const poTokenGenerate = async (
@@ -177,7 +177,9 @@ async function checkToken({
"failed to find valid video with adaptive format to check token against",
);
}

12
patches/0008-add-proxy-retries-on-innertube-error.patch
View file

@ -1,7 +1,7 @@
From a7672c1dba33fceb9c4de722a2ca6b2f56767007 Mon Sep 17 00:00:00 2001
From 980102372e0368917533c1bd8c4b8380b05ee775 Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Tue, 25 Mar 2025 00:04:47 -0300
Subject: [PATCH 08/13] add proxy retries on innertube error
Subject: [PATCH 08/11] add proxy retries on innertube error
---
src/lib/helpers/config.ts | 1 +
@ -9,10 +9,10 @@ Subject: [PATCH 08/13] add proxy retries on innertube error
2 files changed, 23 insertions(+), 1 deletion(-)
diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts
index e6b1070..acbede2 100644
index 6004753..c840ce4 100644
--- a/src/lib/helpers/config.ts
+++ b/src/lib/helpers/config.ts
@@ -37,6 +37,7 @@ const ConfigSchema = z.object({
@@ -40,6 +40,7 @@ export const ConfigSchema = z.object({
external_videoplayback_proxy: z.string().default(
Deno.env.get("EXTERNAL_VIDEOPLAYBACK_PROXY") || "",
),
@ -21,10 +21,10 @@ index e6b1070..acbede2 100644
jobs: z.object({
youtube_session: z.object({
diff --git a/src/lib/helpers/youtubePlayerHandling.ts b/src/lib/helpers/youtubePlayerHandling.ts
index c7c2f74..396eabf 100644
index 4c9ab51..792ba54 100644
--- a/src/lib/helpers/youtubePlayerHandling.ts
+++ b/src/lib/helpers/youtubePlayerHandling.ts
@@ -40,12 +40,33 @@ export const youtubePlayerParsing = async ({
@@ -43,12 +43,33 @@ export const youtubePlayerParsing = async ({
if (videoCached != null && cacheEnabled) {
return JSON.parse(new TextDecoder().decode(decompress(videoCached)));
} else {

14
patches/0010-add-metrics-for-proxy-retries.patch → patches/0009-add-metrics-for-proxy-retries.patch
View file

@ -1,7 +1,7 @@
From dc1df7a313083757fd5b74da3dc1739b0e45eb27 Mon Sep 17 00:00:00 2001
From 7e9c64913cb6085d94e0f2e3977d831573592fc9 Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Tue, 25 Mar 2025 00:07:28 -0300
Subject: [PATCH 10/13] add metrics for proxy retries
Subject: [PATCH 09/11] add metrics for proxy retries
---
src/lib/helpers/metrics.ts | 5 +++++
@ -9,10 +9,10 @@ Subject: [PATCH 10/13] add metrics for proxy retries
2 files changed, 6 insertions(+)
diff --git a/src/lib/helpers/metrics.ts b/src/lib/helpers/metrics.ts
index 98ca83d..93c6b82 100644
index ca972df..5dee540 100644
--- a/src/lib/helpers/metrics.ts
+++ b/src/lib/helpers/metrics.ts
@@ -55,6 +55,11 @@ export class Metrics {
@@ -58,6 +58,11 @@ export class Metrics {
"Number failed requests made to the Innertube API for whatever reason",
);
@ -21,11 +21,11 @@ index 98ca83d..93c6b82 100644
+ 'Times a request to innertube has been retried when it gets "This helps protect our community"',
+ );
+
public checkInnertubeResponse(videoData: IRawResponse) {
this.innertubeFailedRequest.inc();
private checkStatus(videoData: IRawResponse) {
const status = videoData.playabilityStatus?.status;
diff --git a/src/lib/helpers/youtubePlayerHandling.ts b/src/lib/helpers/youtubePlayerHandling.ts
index 2ae878d..7b5f0c1 100644
index 792ba54..9efed59 100644
--- a/src/lib/helpers/youtubePlayerHandling.ts
+++ b/src/lib/helpers/youtubePlayerHandling.ts
@@ -62,6 +62,7 @@ export const youtubePlayerParsing = async ({

2674
patches/0009-add-support-for-prometheus-metrics.patch
View file

File diff suppressed because it is too large Load diff

4
patches/0011-fix-fix-tokio-overflow-on-compile.patch → patches/0010-fix-fix-tokio-overflow-on-compile.patch
View file

@ -1,7 +1,7 @@
From a9cc6d6dc6953ec1ea5bd49cb80e78ed25b6e0af Mon Sep 17 00:00:00 2001
From b35e273dbbb21e872697892bdc2a2352e3161411 Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Tue, 25 Mar 2025 00:24:07 -0300
Subject: [PATCH 11/13] fix: fix tokio overflow on compile
Subject: [PATCH 10/11] fix: fix tokio overflow on compile
---
Dockerfile | 2 ++

8
patches/0012-Add-environment-variable-for-youtube_session.frequen.patch → patches/0011-Add-environment-variable-for-youtube_session.frequen.patch
View file

@ -1,17 +1,17 @@
From 046aa5c93e998ceaba089ab38b6ee5ed7162705c Mon Sep 17 00:00:00 2001
From 0f41fe19781d9ce29b2043fbc8b184197ba2b6bf Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Wed, 26 Mar 2025 12:24:49 -0300
Subject: [PATCH 12/13] Add environment variable for youtube_session.frequency
Subject: [PATCH 11/11] Add environment variable for youtube_session.frequency
---
src/lib/helpers/config.ts | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts
index 50f3f8b..8cf34fd 100644
index c840ce4..6b9170d 100644
--- a/src/lib/helpers/config.ts
+++ b/src/lib/helpers/config.ts
@@ -51,7 +51,9 @@ const ConfigSchema = z.object({
@@ -45,7 +45,9 @@ export const ConfigSchema = z.object({
jobs: z.object({
youtube_session: z.object({
po_token_enabled: z.boolean().default(true),

699
patches/0013-Move-po-token-to-webworker.patch
View file

@ -1,699 +0,0 @@
From 9d4ad948d08584a771995bf06ddf209fc8c97ee7 Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Tue, 1 Apr 2025 19:18:15 -0300
Subject: [PATCH 13/13] Move po token to webworker
https://github.com/iv-org/invidious-companion/pull/73
Squashed commit of the following:
commit 99929b0e8d1e26ca872437e0f1799efa3632dc28
Author: Alex Maras <dev@alexmaras.com>
Date: Wed Mar 26 21:43:25 2025 +0800
chore: remove memory logging
commit 12f7eee268cbc4224b6e2778d7e3abbbac9ae011
Author: Alex Maras <dev@alexmaras.com>
Date: Mon Mar 24 18:14:40 2025 +0800
chore: deno fmt
commit 9befe1b2bf6d2294b1765213bb5be98d649cce06
Author: Alex Maras <dev@alexmaras.com>
Date: Mon Mar 24 18:13:25 2025 +0800
chore: improve typing in worker.ts
commit 186efd73c05980b91c8e81290327cf5f8eaa1629
Author: Alex Maras <dev@alexmaras.com>
Date: Sat Mar 22 10:57:43 2025 +0800
chore: fmt
commit 391f91571a212d46055e35edfaa00b8efc2b5bc1
Author: Alex Maras <dev@alexmaras.com>
Date: Sat Mar 22 10:57:08 2025 +0800
chore: use z.union instead of .or in worker
commit 3df53068293b513b6502f0bbb29d549027411c5a
Author: Alex Maras <dev@alexmaras.com>
Date: Thu Mar 20 14:06:52 2025 +0800
chore: temporary heap memory usage output
commit a1cb9da1ca3cd8890007d9b2ff4b2b400860228a
Author: Alex Maras <dev@alexmaras.com>
Date: Thu Mar 20 14:06:24 2025 +0800
feat: split po token generation to web worker
---
src/lib/helpers/config.ts | 2 +-
src/lib/helpers/youtubePlayerHandling.ts | 4 +-
src/lib/helpers/youtubePlayerReq.ts | 6 +-
src/lib/jobs/potoken.ts | 225 +++++++++++-----------
src/lib/jobs/worker.ts | 231 +++++++++++++++++++++++
src/lib/types/HonoVariables.ts | 4 +-
src/main.ts | 22 +--
7 files changed, 361 insertions(+), 133 deletions(-)
create mode 100644 src/lib/jobs/worker.ts
diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts
index 8cf34fd..2a3fdb3 100644
--- a/src/lib/helpers/config.ts
+++ b/src/lib/helpers/config.ts
@@ -1,7 +1,7 @@
import { z, ZodError } from "zod";
import { parse } from "@std/toml";
-const ConfigSchema = z.object({
+export const ConfigSchema = z.object({
server: z.object({
port: z.number().default(Number(Deno.env.get("PORT")) || 8282),
host: z.string().default(Deno.env.get("HOST") || "127.0.0.1"),
diff --git a/src/lib/helpers/youtubePlayerHandling.ts b/src/lib/helpers/youtubePlayerHandling.ts
index 7b5f0c1..62eb30c 100644
--- a/src/lib/helpers/youtubePlayerHandling.ts
+++ b/src/lib/helpers/youtubePlayerHandling.ts
@@ -1,8 +1,8 @@
import { ApiResponse, Innertube, YT } from "youtubei.js";
import { generateRandomString } from "youtubei.js/Utils";
import { compress, decompress } from "brotli";
-import type { BG } from "bgutils";
import { Metrics } from "../helpers/metrics.ts";
+import type { TokenMinter } from "../jobs/potoken.ts";
let youtubePlayerReqLocation = "youtubePlayerReq";
if (Deno.env.get("YT_PLAYER_REQ_LOCATION")) {
if (Deno.env.has("DENO_COMPILED")) {
@@ -31,7 +31,7 @@ export const youtubePlayerParsing = async ({
innertubeClient: Innertube;
videoId: string;
config: Config;
- tokenMinter: BG.WebPoMinter;
+ tokenMinter: TokenMinter;
metrics: Metrics | undefined;
overrideCache?: boolean;
}): Promise<object> => {
diff --git a/src/lib/helpers/youtubePlayerReq.ts b/src/lib/helpers/youtubePlayerReq.ts
index 884fae4..af78269 100644
--- a/src/lib/helpers/youtubePlayerReq.ts
+++ b/src/lib/helpers/youtubePlayerReq.ts
@@ -1,6 +1,6 @@
import { ApiResponse, Innertube } from "youtubei.js";
import NavigationEndpoint from "youtubei.js/NavigationEndpoint";
-import type { BG } from "bgutils";
+import type { TokenMinter } from "../jobs/potoken.ts";
import type { Config } from "./config.ts";
@@ -8,7 +8,7 @@ export const youtubePlayerReq = async (
innertubeClient: Innertube,
videoId: string,
config: Config,
- tokenMinter: BG.WebPoMinter,
+ tokenMinter: TokenMinter,
): Promise<ApiResponse> => {
const innertubeClientOauthEnabled = config.youtube_session.oauth_enabled;
@@ -21,7 +21,7 @@ export const youtubePlayerReq = async (
watchEndpoint: { videoId: videoId },
});
- const contentPoToken = await tokenMinter.mintAsWebsafeString(videoId);
+ const contentPoToken = await tokenMinter(videoId);
return watch_endpoint.call(innertubeClient.actions, {
playbackContext: {
diff --git a/src/lib/jobs/potoken.ts b/src/lib/jobs/potoken.ts
index e7da3c0..fe99e9c 100644
--- a/src/lib/jobs/potoken.ts
+++ b/src/lib/jobs/potoken.ts
@@ -1,6 +1,3 @@
-import { BG, buildURL, GOOG_API_KEY, USER_AGENT } from "bgutils";
-import type { WebPoSignalOutput } from "bgutils";
-import { JSDOM } from "jsdom";
import { Innertube, UniversalCache } from "youtubei.js";
import {
youtubePlayerParsing,
@@ -21,121 +18,134 @@ if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
}
const { getFetchClient } = await import(getFetchClientLocation);
-// Adapted from https://github.com/LuanRT/BgUtils/blob/main/examples/node/index.ts
-export const poTokenGenerate = async (
- innertubeClient: Innertube,
- config: Config,
- innertubeClientCache: UniversalCache,
- metrics: Metrics | undefined,
-): Promise<{ innertubeClient: Innertube; tokenMinter: BG.WebPoMinter }> => {
- if (innertubeClient.session.po_token) {
- innertubeClient = await Innertube.create({
- enable_session_cache: false,
- user_agent: USER_AGENT,
- retrieve_player: false,
- });
- }
+import { InputMessage, OutputMessageSchema } from "./worker.ts";
- const fetchImpl = await getFetchClient(config);
-
- const visitorData = innertubeClient.session.context.client.visitorData;
-
- if (!visitorData) {
- throw new Error("Could not get visitor data");
- }
-
- const dom = new JSDOM(
- '<!DOCTYPE html><html lang="en"><head><title></title></head><body></body></html>',
- {
- url: "https://www.youtube.com/",
- referrer: "https://www.youtube.com/",
- userAgent: USER_AGENT,
- },
- );
-
- Object.assign(globalThis, {
- window: dom.window,
- document: dom.window.document,
- location: dom.window.location,
- origin: dom.window.origin,
- });
+interface TokenGeneratorWorker extends Omit<Worker, "postMessage"> {
+ postMessage(message: InputMessage): void;
+}
- if (!Reflect.has(globalThis, "navigator")) {
- Object.defineProperty(globalThis, "navigator", {
- value: dom.window.navigator,
+const workers: TokenGeneratorWorker[] = [];
+
+function createMinter(worker: TokenGeneratorWorker) {
+ return (videoId: string): Promise<string> => {
+ const { promise, resolve } = Promise.withResolvers<string>();
+ // generate a UUID to identify the request as many minter calls
+ // may be made within a timespan, and this function will be
+ // informed about all of them until it's got its own
+ const requestId = crypto.randomUUID();
+ const listener = (message: MessageEvent) => {
+ const parsedMessage = OutputMessageSchema.parse(message.data);
+ if (
+ parsedMessage.type === "content-token" &&
+ parsedMessage.requestId === requestId
+ ) {
+ worker.removeEventListener("message", listener);
+ resolve(parsedMessage.contentToken);
+ }
+ };
+ worker.addEventListener("message", listener);
+ worker.postMessage({
+ type: "content-token-request",
+ videoId,
+ requestId,
});
- }
-
- const challengeResponse = await innertubeClient.getAttestationChallenge(
- "ENGAGEMENT_TYPE_UNBOUND",
- );
- if (!challengeResponse.bg_challenge) {
- throw new Error("Could not get challenge");
- }
-
- const interpreterUrl = challengeResponse.bg_challenge.interpreter_url
- .private_do_not_access_or_else_trusted_resource_url_wrapped_value;
- const bgScriptResponse = await fetchImpl(
- `http:${interpreterUrl}`,
- );
- const interpreterJavascript = await bgScriptResponse.text();
- if (interpreterJavascript) {
- new Function(interpreterJavascript)();
- } else throw new Error("Could not load VM");
+ return promise;
+ };
+}
- // Botguard currently surfaces a "Not implemented" error here, due to the environment
- // not having a valid Canvas API in JSDOM. At the time of writing, this doesn't cause
- // any issues as the Canvas check doesn't appear to be an enforced element of the checks
- console.log(
- '[INFO] the "Not implemented: HTMLCanvasElement.prototype.getContext" error is normal. Please do not open a bug report about it.',
- );
- const botguard = await BG.BotGuardClient.create({
- program: challengeResponse.bg_challenge.program,
- globalName: challengeResponse.bg_challenge.global_name,
- globalObj: globalThis,
- });
+export type TokenMinter = ReturnType<typeof createMinter>;
- const webPoSignalOutput: WebPoSignalOutput = [];
- const botguardResponse = await botguard.snapshot({ webPoSignalOutput });
- const requestKey = "O43z0dpjhgX20SCx4KAo";
+// Adapted from https://github.com/LuanRT/BgUtils/blob/main/examples/node/index.ts
+export const poTokenGenerate = (
+ config: Config,
+ innertubeClientCache: UniversalCache,
+ metrics: Metrics | undefined,
+): Promise<{ innertubeClient: Innertube; tokenMinter: TokenMinter }> => {
+ const { promise, resolve, reject } = Promise.withResolvers<
+ Awaited<ReturnType<typeof poTokenGenerate>>
+ >();
- const integrityTokenResponse = await fetchImpl(
- buildURL("GenerateIT", true),
+ const worker: TokenGeneratorWorker = new Worker(
+ new URL("./worker.ts", import.meta.url).href,
{
- method: "POST",
- headers: {
- "content-type": "application/json+protobuf",
- "x-goog-api-key": GOOG_API_KEY,
- "x-user-agent": "grpc-web-javascript/0.1",
- "user-agent": USER_AGENT,
- },
- body: JSON.stringify([requestKey, botguardResponse]),
+ type: "module",
+ name: "PO Token Generator",
},
);
+ // take note of the worker so we can kill it once a new one takes its place
+ workers.push(worker);
+ worker.addEventListener("message", async (event) => {
+ const parsedMessage = OutputMessageSchema.parse(event.data);
+
+ // worker is listening for messages
+ if (parsedMessage.type === "ready") {
+ const untypedPostMessage = worker.postMessage.bind(worker);
+ worker.postMessage = (message: InputMessage) =>
+ untypedPostMessage(message);
+ worker.postMessage({ type: "initialise", config });
+ }
- const response = await integrityTokenResponse.json() as unknown[];
-
- if (typeof response[0] !== "string") {
- throw new Error("Could not get integrity token");
- }
+ if (parsedMessage.type === "error") {
+ console.log({ errorFromWorker: parsedMessage.error });
+ worker.terminate();
+ reject(parsedMessage.error);
+ }
- const integrityTokenBasedMinter = await BG.WebPoMinter.create({
- integrityToken: response[0],
- }, webPoSignalOutput);
+ // worker is initialised and has passed back a session token and visitor data
+ if (parsedMessage.type === "initialised") {
+ try {
+ const instantiatedInnertubeClient = await Innertube.create({
+ enable_session_cache: false,
+ po_token: parsedMessage.sessionPoToken,
+ visitor_data: parsedMessage.visitorData,
+ fetch: getFetchClient(config),
+ cache: innertubeClientCache,
+ generate_session_locally: true,
+ });
+ const minter = createMinter(worker);
+ // check token from minter
+ await checkToken({
+ instantiatedInnertubeClient,
+ config,
+ integrityTokenBasedMinter: minter,
+ metrics,
+ });
+ console.log("Successfully generated PO token");
+ const numberToKill = workers.length - 1;
+ for (let i = 0; i < numberToKill; i++) {
+ const workerToKill = workers.shift();
+ workerToKill?.terminate();
+ }
+ return resolve({
+ innertubeClient: instantiatedInnertubeClient,
+ tokenMinter: minter,
+ });
+ } catch (err) {
+ console.log("Failed to get valid PO token, will retry", {
+ err,
+ });
+ worker.terminate();
+ reject(err);
+ }
+ }
+ });
- const sessionPoToken = await integrityTokenBasedMinter.mintAsWebsafeString(
- visitorData,
- );
+ return promise;
+};
- const instantiatedInnertubeClient = await Innertube.create({
- enable_session_cache: false,
- po_token: sessionPoToken,
- visitor_data: visitorData,
- fetch: getFetchClient(config),
- cache: innertubeClientCache,
- generate_session_locally: true,
- });
+async function checkToken({
+ instantiatedInnertubeClient,
+ config,
+ integrityTokenBasedMinter,
+ metrics,
+}: {
+ instantiatedInnertubeClient: Innertube;
+ config: Config;
+ integrityTokenBasedMinter: TokenMinter;
+ metrics: Metrics | undefined,
+}) {
+ const fetchImpl = getFetchClient(config);
if (config.jobs.youtube_session.po_token_check) {
try {
@@ -183,9 +193,4 @@ export const poTokenGenerate = async (
throw err;
}
}
-
- return {
- innertubeClient: instantiatedInnertubeClient,
- tokenMinter: integrityTokenBasedMinter,
- };
-};
+}
diff --git a/src/lib/jobs/worker.ts b/src/lib/jobs/worker.ts
new file mode 100644
index 0000000..0d62bfc
--- /dev/null
+++ b/src/lib/jobs/worker.ts
@@ -0,0 +1,231 @@
+/// <reference lib="webworker" />
+
+import { z } from "zod";
+import { Config, ConfigSchema } from "../helpers/config.ts";
+import { BG, buildURL, GOOG_API_KEY, USER_AGENT } from "bgutils";
+import type { WebPoSignalOutput } from "bgutils";
+import { JSDOM } from "jsdom";
+import { Innertube } from "youtubei.js";
+let getFetchClientLocation = "getFetchClient";
+if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
+ if (Deno.env.has("DENO_COMPILED")) {
+ getFetchClientLocation = Deno.mainModule.replace("src/main.ts", "") +
+ Deno.env.get("GET_FETCH_CLIENT_LOCATION");
+ } else {
+ getFetchClientLocation = Deno.env.get(
+ "GET_FETCH_CLIENT_LOCATION",
+ ) as string;
+ }
+}
+
+type FetchFunction = typeof fetch;
+const { getFetchClient }: {
+ getFetchClient: (config: Config) => Promise<FetchFunction>;
+} = await import(getFetchClientLocation);
+
+// ---- Messages to send to the webworker ----
+const InputInitialiseSchema = z.object({
+ type: z.literal("initialise"),
+ config: ConfigSchema,
+}).strict();
+
+const InputContentTokenSchema = z.object({
+ type: z.literal("content-token-request"),
+ videoId: z.string(),
+ requestId: z.string().uuid(),
+}).strict();
+export type InputInitialise = z.infer<typeof InputInitialiseSchema>;
+export type InputContentToken = z.infer<typeof InputContentTokenSchema>;
+const InputMessageSchema = z.union([
+ InputInitialiseSchema,
+ InputContentTokenSchema,
+]);
+export type InputMessage = z.infer<typeof InputMessageSchema>;
+
+// ---- Messages that the webworker sends to the parent ----
+const OutputReadySchema = z.object({
+ type: z.literal("ready"),
+}).strict();
+
+const OutputInitialiseSchema = z.object({
+ type: z.literal("initialised"),
+ sessionPoToken: z.string(),
+ visitorData: z.string(),
+}).strict();
+
+const OutputContentTokenSchema = z.object({
+ type: z.literal("content-token"),
+ contentToken: z.string(),
+ requestId: InputContentTokenSchema.shape.requestId,
+}).strict();
+
+const OutputErrorSchema = z.object({
+ type: z.literal("error"),
+ error: z.any(),
+}).strict();
+export const OutputMessageSchema = z.union([
+ OutputReadySchema,
+ OutputInitialiseSchema,
+ OutputContentTokenSchema,
+ OutputErrorSchema,
+]);
+type OutputMessage = z.infer<typeof OutputMessageSchema>;
+
+const IntegrityTokenResponse = z.tuple([z.string()]).rest(z.any());
+
+const isWorker = typeof WorkerGlobalScope !== "undefined" &&
+ self instanceof WorkerGlobalScope;
+if (isWorker) {
+ // helper function to force type-checking
+ const untypedPostmessage = self.postMessage.bind(self);
+ const postMessage = (message: OutputMessage) => {
+ untypedPostmessage(message);
+ };
+
+ let minter: BG.WebPoMinter;
+
+ onmessage = async (event) => {
+ const message = InputMessageSchema.parse(event.data);
+ if (message.type === "initialise") {
+ const fetchImpl: typeof fetch = await getFetchClient(
+ message.config,
+ );
+ try {
+ const {
+ sessionPoToken,
+ visitorData,
+ generatedMinter,
+ } = await setup({ fetchImpl });
+ minter = generatedMinter;
+ postMessage({
+ type: "initialised",
+ sessionPoToken,
+ visitorData,
+ });
+ } catch (err) {
+ postMessage({ type: "error", error: err });
+ }
+ }
+ // this is called every time a video needs a content token
+ if (message.type === "content-token-request") {
+ if (!minter) {
+ throw new Error(
+ "Minter not yet ready, must initialise first",
+ );
+ }
+ const contentToken = await minter.mintAsWebsafeString(
+ message.videoId,
+ );
+ postMessage({
+ type: "content-token",
+ contentToken,
+ requestId: message.requestId,
+ });
+ }
+ };
+
+ postMessage({ type: "ready" });
+}
+
+async function setup(
+ { fetchImpl }: { fetchImpl: FetchFunction },
+) {
+ const innertubeClient = await Innertube.create({
+ enable_session_cache: false,
+ user_agent: USER_AGENT,
+ retrieve_player: false,
+ });
+
+ const visitorData = innertubeClient.session.context.client.visitorData;
+
+ if (!visitorData) {
+ throw new Error("Could not get visitor data");
+ }
+
+ const dom = new JSDOM(
+ '<!DOCTYPE html><html lang="en"><head><title></title></head><body></body></html>',
+ {
+ url: "https://www.youtube.com/",
+ referrer: "https://www.youtube.com/",
+ userAgent: USER_AGENT,
+ },
+ );
+
+ Object.assign(globalThis, {
+ window: dom.window,
+ document: dom.window.document,
+ // location: dom.window.location, // --- doesn't seem to be necessary and the Web Worker doesn't like it
+ origin: dom.window.origin,
+ });
+
+ if (!Reflect.has(globalThis, "navigator")) {
+ Object.defineProperty(globalThis, "navigator", {
+ value: dom.window.navigator,
+ });
+ }
+
+ const challengeResponse = await innertubeClient.getAttestationChallenge(
+ "ENGAGEMENT_TYPE_UNBOUND",
+ );
+ if (!challengeResponse.bg_challenge) {
+ throw new Error("Could not get challenge");
+ }
+
+ const interpreterUrl = challengeResponse.bg_challenge.interpreter_url
+ .private_do_not_access_or_else_trusted_resource_url_wrapped_value;
+ const bgScriptResponse = await fetchImpl(
+ `https:${interpreterUrl}`,
+ );
+ const interpreterJavascript = await bgScriptResponse.text();
+
+ if (interpreterJavascript) {
+ new Function(interpreterJavascript)();
+ } else throw new Error("Could not load VM");
+
+ // Botguard currently surfaces a "Not implemented" error here, due to the environment
+ // not having a valid Canvas API in JSDOM. At the time of writing, this doesn't cause
+ // any issues as the Canvas check doesn't appear to be an enforced element of the checks
+ console.log(
+ '[INFO] the "Not implemented: HTMLCanvasElement.prototype.getContext" error is normal. Please do not open a bug report about it.',
+ );
+ const botguard = await BG.BotGuardClient.create({
+ program: challengeResponse.bg_challenge.program,
+ globalName: challengeResponse.bg_challenge.global_name,
+ globalObj: globalThis,
+ });
+
+ const webPoSignalOutput: WebPoSignalOutput = [];
+ const botguardResponse = await botguard.snapshot({ webPoSignalOutput });
+ const requestKey = "O43z0dpjhgX20SCx4KAo";
+
+ const integrityTokenResponse = await fetchImpl(
+ buildURL("GenerateIT", true),
+ {
+ method: "POST",
+ headers: {
+ "content-type": "application/json+protobuf",
+ "x-goog-api-key": GOOG_API_KEY,
+ "x-user-agent": "grpc-web-javascript/0.1",
+ "user-agent": USER_AGENT,
+ },
+ body: JSON.stringify([requestKey, botguardResponse]),
+ },
+ );
+ const integrityTokenBody = IntegrityTokenResponse.parse(
+ await integrityTokenResponse.json(),
+ );
+
+ const integrityTokenBasedMinter = await BG.WebPoMinter.create({
+ integrityToken: integrityTokenBody[0],
+ }, webPoSignalOutput);
+
+ const sessionPoToken = await integrityTokenBasedMinter.mintAsWebsafeString(
+ visitorData,
+ );
+
+ return {
+ sessionPoToken,
+ visitorData,
+ generatedMinter: integrityTokenBasedMinter,
+ };
+}
diff --git a/src/lib/types/HonoVariables.ts b/src/lib/types/HonoVariables.ts
index 1e9b7a2..ab385ab 100644
--- a/src/lib/types/HonoVariables.ts
+++ b/src/lib/types/HonoVariables.ts
@@ -1,11 +1,11 @@
import { Innertube } from "youtubei.js";
-import { BG } from "bgutils";
+import type { TokenMinter } from "../jobs/potoken.ts";
import type { Config } from "../helpers/config.ts";
import { Metrics } from "../helpers/metrics.ts";
export type HonoVariables = {
innertubeClient: Innertube;
config: Config;
- tokenMinter: BG.WebPoMinter;
+ tokenMinter: TokenMinter;
metrics: Metrics | undefined;
};
diff --git a/src/main.ts b/src/main.ts
index 851cd71..cc9e97d 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -1,10 +1,9 @@
import { Hono } from "hono";
import { routes } from "./routes/index.ts";
import { Innertube, UniversalCache } from "youtubei.js";
-import { poTokenGenerate } from "./lib/jobs/potoken.ts";
+import { poTokenGenerate, type TokenMinter } from "./lib/jobs/potoken.ts";
import { USER_AGENT } from "bgutils";
import { retry } from "@std/async";
-import type { BG } from "bgutils";
import type { HonoVariables } from "./lib/types/HonoVariables.ts";
import { parseConfig } from "./lib/helpers/config.ts";
@@ -30,7 +29,7 @@ declare module "hono" {
const app = new Hono();
const metrics = config.server.enable_metrics ? new Metrics() : undefined;
-let tokenMinter: BG.WebPoMinter;
+let tokenMinter: TokenMinter;
let innertubeClient: Innertube;
let innertubeClientFetchPlayer = true;
const innertubeClientOauthEnabled = config.youtube_session.oauth_enabled;
@@ -65,7 +64,6 @@ if (!innertubeClientOauthEnabled) {
({ innertubeClient, tokenMinter } = await retry(
poTokenGenerate.bind(
poTokenGenerate,
- innertubeClient,
config,
innertubeClientCache,
metrics,
@@ -79,17 +77,11 @@ if (!innertubeClientOauthEnabled) {
{ backoffSchedule: [5_000, 15_000, 60_000, 180_000] },
async () => {
if (innertubeClientJobPoTokenEnabled) {
- try {
- ({ innertubeClient, tokenMinter } = await poTokenGenerate(
- innertubeClient,
- config,
- innertubeClientCache,
- metrics,
- ));
- } catch (err) {
- metrics?.potokenGenerationFailure.inc();
- throw err;
- }
+ ({ innertubeClient, tokenMinter } = await poTokenGenerate(
+ config,
+ innertubeClientCache,
+ metrics,
+ ));
} else {
innertubeClient = await Innertube.create({
enable_session_cache: false,
--
2.49.0