diff --git a/patches/0001-feat-add-support-for-an-external-videoplayback-proxy.patch b/patches/0001-feat-add-support-for-an-external-videoplayback-proxy.patch index 1b94daa..0343833 100644 --- a/patches/0001-feat-add-support-for-an-external-videoplayback-proxy.patch +++ b/patches/0001-feat-add-support-for-an-external-videoplayback-proxy.patch @@ -1,7 +1,7 @@ -From 36964b3308ae64fa262cf75a3139f72ec0b4ddb6 Mon Sep 17 00:00:00 2001 +From 8845d9d0c7c3548d56700dd844976e792bed954b Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 18:44:10 -0300 -Subject: [PATCH 01/10] feat: add support for an external videoplayback proxy +Subject: [PATCH 01/13] feat: add support for an external videoplayback proxy --- config/config.example.toml | 1 + diff --git a/patches/0002-feat-report-the-external-videoplayback-proxy-via-inf.patch b/patches/0002-feat-report-the-external-videoplayback-proxy-via-inf.patch index 7e7af25..7ffab2f 100644 --- a/patches/0002-feat-report-the-external-videoplayback-proxy-via-inf.patch +++ b/patches/0002-feat-report-the-external-videoplayback-proxy-via-inf.patch @@ -1,7 +1,7 @@ -From f417f917ae71707e024af334047818beaf85b032 Mon Sep 17 00:00:00 2001 +From 9f61e94d874a07e1c05db2adada3877f73ffa3ec Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 18:52:53 -0300 -Subject: [PATCH 02/10] feat: report the external videoplayback proxy via /info +Subject: [PATCH 02/13] feat: report the external videoplayback proxy via /info endpoint --- diff --git a/patches/0003-feat-add-resolution-limit-on-DASH-streams-to-save-ba.patch b/patches/0003-feat-add-resolution-limit-on-DASH-streams-to-save-ba.patch index e5e2e93..b1e9af4 100644 --- a/patches/0003-feat-add-resolution-limit-on-DASH-streams-to-save-ba.patch +++ b/patches/0003-feat-add-resolution-limit-on-DASH-streams-to-save-ba.patch @@ -1,7 +1,7 @@ -From 5b4ddd3e53d47638b12e34089ebcda72566fc154 Mon Sep 17 00:00:00 2001 +From 54041ba8e3837b9f8f1b7350d16cd581b48b8208 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 19:02:01 -0300 -Subject: [PATCH 03/10] feat: add resolution limit on DASH streams to save +Subject: [PATCH 03/13] feat: add resolution limit on DASH streams to save bandwidth --- diff --git a/patches/0004-feat-add-env-variable-to-set-verify_requests.patch b/patches/0004-feat-add-env-variable-to-set-verify_requests.patch index fce8201..be7377b 100644 --- a/patches/0004-feat-add-env-variable-to-set-verify_requests.patch +++ b/patches/0004-feat-add-env-variable-to-set-verify_requests.patch @@ -1,7 +1,7 @@ -From 1c1ee023f629036e0c391f4baea9bf7d5863dd8f Mon Sep 17 00:00:00 2001 +From 711cdeb0a284915496602e1d787c21aa57d36f8f Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 19:06:04 -0300 -Subject: [PATCH 04/10] feat: add env variable to set verify_requests +Subject: [PATCH 04/13] feat: add env variable to set verify_requests --- src/lib/helpers/config.ts | 4 +++- diff --git a/patches/0005-feat-add-support-for-multiple-proxies.patch b/patches/0005-feat-add-support-for-multiple-proxies.patch index 8ddb141..77a19c1 100644 --- a/patches/0005-feat-add-support-for-multiple-proxies.patch +++ b/patches/0005-feat-add-support-for-multiple-proxies.patch @@ -1,7 +1,7 @@ -From 855b0ebc1e072442c9c12d8cb56fd8edba177e23 Mon Sep 17 00:00:00 2001 +From dbff87e223c996f47d0c38d7edc4e87288e428b6 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 19:20:52 -0300 -Subject: [PATCH 05/10] feat: add support for multiple proxies +Subject: [PATCH 05/13] feat: add support for multiple proxies --- src/lib/helpers/getFetchClient.ts | 17 ++++++++++++++++- diff --git a/patches/0006-ci-update-deno-to-2.2.4.patch b/patches/0006-ci-update-deno-to-2.2.4.patch index 33f91b1..407aee1 100644 --- a/patches/0006-ci-update-deno-to-2.2.4.patch +++ b/patches/0006-ci-update-deno-to-2.2.4.patch @@ -1,7 +1,7 @@ -From 7b46627077debdd473e50f3c7cf2ccf9a98d4bd3 Mon Sep 17 00:00:00 2001 +From 84de478e016b136f7096c5cfb71d9fb5ce05f6af Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 19:37:34 -0300 -Subject: [PATCH 06/10] ci: update deno to 2.2.4 +Subject: [PATCH 06/13] ci: update deno to 2.2.4 --- Dockerfile | 2 +- diff --git a/patches/0007-fix-temporary-player_id-override-until-an-official-f.patch b/patches/0007-fix-temporary-player_id-override-until-an-official-f.patch index 538cff7..07cc2ed 100644 --- a/patches/0007-fix-temporary-player_id-override-until-an-official-f.patch +++ b/patches/0007-fix-temporary-player_id-override-until-an-official-f.patch @@ -1,7 +1,7 @@ -From 350c0637f2a6c6fb45901b5f451ab7bc86761db4 Mon Sep 17 00:00:00 2001 +From 7ba83928b7bbb3b27edfaa35dd8c82a68ea977c9 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 20:06:47 -0300 -Subject: [PATCH 07/10] fix: temporary player_id override until an official fix +Subject: [PATCH 07/13] fix: temporary player_id override until an official fix cames out --- diff --git a/patches/0008-fix-cut-off-secret_key-to-16-characters.patch b/patches/0008-fix-cut-off-secret_key-to-16-characters.patch index 4836536..9a17ecb 100644 --- a/patches/0008-fix-cut-off-secret_key-to-16-characters.patch +++ b/patches/0008-fix-cut-off-secret_key-to-16-characters.patch @@ -1,7 +1,7 @@ -From 2250607fe6e490c9918560eab48c807257a0d9ed Mon Sep 17 00:00:00 2001 +From 5b1142026dbf518e024500c68e1e2f3976f5c25e Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 20:22:19 -0300 -Subject: [PATCH 08/10] fix: cut off secret_key to 16 characters +Subject: [PATCH 08/13] fix: cut off secret_key to 16 characters --- src/lib/helpers/verifyRequest.ts | 2 +- diff --git a/patches/0009-feat-add-option-to-disable-potoken-generation-check.patch b/patches/0009-feat-add-option-to-disable-potoken-generation-check.patch index b3e5ceb..ad0d74b 100644 --- a/patches/0009-feat-add-option-to-disable-potoken-generation-check.patch +++ b/patches/0009-feat-add-option-to-disable-potoken-generation-check.patch @@ -1,7 +1,7 @@ -From 9a955b474825e81164e4dc56294b8822dace8c1d Mon Sep 17 00:00:00 2001 +From b85d4506c03a78d5c216a3add1f6ad1d43fbddfe Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 20:34:33 -0300 -Subject: [PATCH 09/10] feat: add option to disable potoken generation check +Subject: [PATCH 09/13] feat: add option to disable potoken generation check --- config/config.example.toml | 1 + diff --git a/patches/0010-feat-add-support-for-encrypted-query-parameters.patch b/patches/0010-feat-add-support-for-encrypted-query-parameters.patch index b628d12..20a790b 100644 --- a/patches/0010-feat-add-support-for-encrypted-query-parameters.patch +++ b/patches/0010-feat-add-support-for-encrypted-query-parameters.patch @@ -1,7 +1,7 @@ -From 8a67c778bb2dd889a32b0b6921a6decf07b71ec0 Mon Sep 17 00:00:00 2001 +From ddda50ee041d1ca71fd0cf3dc0f91e29d9ede8e5 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 21:38:33 -0300 -Subject: [PATCH 10/10] feat: add support for encrypted query parameters +Subject: [PATCH 10/13] feat: add support for encrypted query parameters --- src/lib/helpers/config.ts | 9 ++++ diff --git a/patches/0011-add-proxy-retries-on-innertube-error.patch b/patches/0011-add-proxy-retries-on-innertube-error.patch new file mode 100644 index 0000000..ac46f70 --- /dev/null +++ b/patches/0011-add-proxy-retries-on-innertube-error.patch @@ -0,0 +1,64 @@ +From df81598e368783e8309577973f63ca81c0490868 Mon Sep 17 00:00:00 2001 +From: Fijxu +Date: Tue, 25 Mar 2025 00:04:47 -0300 +Subject: [PATCH 11/13] add proxy retries on innertube error + +--- + src/lib/helpers/config.ts | 1 + + src/lib/helpers/youtubePlayerHandling.ts | 23 ++++++++++++++++++++++- + 2 files changed, 23 insertions(+), 1 deletion(-) + +diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts +index 83ce0fa..d539738 100644 +--- a/src/lib/helpers/config.ts ++++ b/src/lib/helpers/config.ts +@@ -43,6 +43,7 @@ const ConfigSchema = z.object({ + external_videoplayback_proxy: z.string().default( + Deno.env.get("EXTERNAL_VIDEOPLAYBACK_PROXY") || "", + ), ++ max_proxy_retries: z.number().default(Number(Deno.env.get("MAX_PROXY_RETIRES") || 5)), + }).strict().default({}), + jobs: z.object({ + youtube_session: z.object({ +diff --git a/src/lib/helpers/youtubePlayerHandling.ts b/src/lib/helpers/youtubePlayerHandling.ts +index c7c2f74..396eabf 100644 +--- a/src/lib/helpers/youtubePlayerHandling.ts ++++ b/src/lib/helpers/youtubePlayerHandling.ts +@@ -40,12 +40,33 @@ export const youtubePlayerParsing = async ({ + if (videoCached != null && cacheEnabled) { + return JSON.parse(new TextDecoder().decode(decompress(videoCached))); + } else { +- const youtubePlayerResponse = await youtubePlayerReq( ++ let youtubePlayerResponse = await youtubePlayerReq( + innertubeClient, + videoId, + config, + tokenMinter, + ); ++ ++ const maxRetries = config.networking.max_proxy_retries; ++ for (let retries = 1; retries <= (maxRetries as number); retries++) { ++ if ( ++ !youtubePlayerResponse.data.playabilityStatus?.errorScreen ++ ?.playerErrorMessageRenderer?.subreason?.runs?.[0]?.text ++ ?.includes("This helps protect our community") ++ ) { ++ break; ++ } ++ console.log( ++ `[DEBUG] Got 'This helps protect our community', retrying request for ${videoId}. Retry ${retries} of ${maxRetries}`, ++ ); ++ youtubePlayerResponse = await youtubePlayerReq( ++ innertubeClient, ++ videoId, ++ config, ++ tokenMinter, ++ ); ++ } ++ + const videoData = youtubePlayerResponse.data; + + const video = new YT.VideoInfo( +-- +2.49.0 + diff --git a/patches/0012-add-support-for-prometheus-metrics.patch b/patches/0012-add-support-for-prometheus-metrics.patch new file mode 100644 index 0000000..69d898c --- /dev/null +++ b/patches/0012-add-support-for-prometheus-metrics.patch @@ -0,0 +1,453 @@ +From af90ccd9de22acefec3a7a035a6a2e4e7fc309fb Mon Sep 17 00:00:00 2001 +From: Fijxu +Date: Tue, 18 Mar 2025 16:38:23 -0300 +Subject: [PATCH 12/13] add support for prometheus metrics + +fix deno lint and typo + +chore: fmt + +apply suggestions + +better innertube response checking + +rename route _metrics to metrics + +use Hono context to pass Metrics onto functions + +fix: add missing metrics argument +--- + config/config.example.toml | 1 + + deno.json | 1 + + deno.lock | 25 ++++- + src/lib/helpers/config.ts | 11 ++- + src/lib/helpers/metrics.ts | 112 +++++++++++++++++++++++ + src/lib/helpers/youtubePlayerHandling.ts | 32 ++++--- + src/lib/types/HonoVariables.ts | 2 + + src/main.ts | 18 +++- + src/routes/index.ts | 4 + + src/routes/metrics.ts | 11 +++ + src/routes/youtube_api_routes/player.ts | 2 + + 11 files changed, 198 insertions(+), 21 deletions(-) + create mode 100644 src/lib/helpers/metrics.ts + create mode 100644 src/routes/metrics.ts + +diff --git a/config/config.example.toml b/config/config.example.toml +index 4c04ee3..01c3f83 100644 +--- a/config/config.example.toml ++++ b/config/config.example.toml +@@ -16,6 +16,7 @@ + # secret_key = "CHANGE_ME" # env variable: SERVER_SECRET_KEY + # verify_requests = false + # max_dash_resolution = 1080 ++# enable_metrics = false # env variable: ENABLE_METRICS + + # [cache] + # enabled = true +diff --git a/deno.json b/deno.json +index 02ea3a2..97600ec 100644 +--- a/deno.json ++++ b/deno.json +@@ -6,6 +6,7 @@ + "imports": { + "hono": "jsr:@hono/hono@4.7.4", + "@std/toml": "jsr:@std/toml@1.0.2", ++ "prom-client": "npm:prom-client@15.1.3", + "youtubei.js": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno.ts", + "youtubei.js/Utils": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/utils/Utils.ts", + "youtubei.js/NavigationEndpoint": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/parser/classes/NavigationEndpoint.ts", +diff --git a/deno.lock b/deno.lock +index 27d8ae6..a9c62e7 100644 +--- a/deno.lock ++++ b/deno.lock +@@ -15,7 +15,8 @@ + "npm:@bufbuild/protobuf@2": "2.2.4", + "npm:@types/estree@^1.0.6": "1.0.6", + "npm:acorn@^8.8.0": "8.14.1", +- "npm:jsdom@26.0.0": "26.0.0" ++ "npm:jsdom@26.0.0": "26.0.0", ++ "npm:prom-client@15.1.3": "15.1.3" + }, + "jsr": { + "@hono/hono@4.7.4": { +@@ -101,6 +102,9 @@ + "@csstools/css-tokenizer@3.0.3": { + "integrity": "sha512-UJnjoFsmxfKUdNYdWgOB0mWUypuLvAfQPH1+pyvRJs6euowbFkFC6P13w1l8mJyi3vxYMxc9kld5jZEGRQs6bw==" + }, ++ "@opentelemetry/api@1.9.0": { ++ "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==" ++ }, + "@types/estree@1.0.6": { + "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==" + }, +@@ -113,6 +117,9 @@ + "asynckit@0.4.0": { + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, ++ "bintrees@1.0.2": { ++ "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" ++ }, + "call-bind-apply-helpers@1.0.2": { + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dependencies": [ +@@ -318,6 +325,13 @@ + "entities" + ] + }, ++ "prom-client@15.1.3": { ++ "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", ++ "dependencies": [ ++ "@opentelemetry/api", ++ "tdigest" ++ ] ++ }, + "punycode@2.3.1": { + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==" + }, +@@ -336,6 +350,12 @@ + "symbol-tree@3.2.4": { + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" + }, ++ "tdigest@0.1.2": { ++ "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", ++ "dependencies": [ ++ "bintrees" ++ ] ++ }, + "tldts-core@6.1.84": { + "integrity": "sha512-NaQa1W76W2aCGjXybvnMYzGSM4x8fvG2AN/pla7qxcg0ZHbooOPhA8kctmOZUDfZyhDL27OGNbwAeig8P4p1vg==" + }, +@@ -1102,7 +1122,8 @@ + "jsr:@std/fs@1.0.14", + "jsr:@std/path@1.0.8", + "jsr:@std/toml@1.0.2", +- "npm:jsdom@26.0.0" ++ "npm:jsdom@26.0.0", ++ "npm:prom-client@15.1.3" + ] + } + } +diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts +index d539738..646132f 100644 +--- a/src/lib/helpers/config.ts ++++ b/src/lib/helpers/config.ts +@@ -23,6 +23,13 @@ const ConfigSchema = z.object({ + ? false + : true, + ), ++ enable_metrics: z.boolean().default( ++ Deno.env.get("ENABLE_METRICS") === "true" ++ ? true ++ : Deno.env.get("ENABLE_METRICS") === "false" ++ ? false ++ : true, ++ ), + }).strict().default({}), + cache: z.object({ + enabled: z.boolean().default(true), +@@ -43,7 +50,9 @@ const ConfigSchema = z.object({ + external_videoplayback_proxy: z.string().default( + Deno.env.get("EXTERNAL_VIDEOPLAYBACK_PROXY") || "", + ), +- max_proxy_retries: z.number().default(Number(Deno.env.get("MAX_PROXY_RETIRES") || 5)), ++ max_proxy_retries: z.number().default( ++ Number(Deno.env.get("MAX_PROXY_RETIRES") || 5), ++ ), + }).strict().default({}), + jobs: z.object({ + youtube_session: z.object({ +diff --git a/src/lib/helpers/metrics.ts b/src/lib/helpers/metrics.ts +new file mode 100644 +index 0000000..98ca83d +--- /dev/null ++++ b/src/lib/helpers/metrics.ts +@@ -0,0 +1,112 @@ ++import { IRawResponse } from "youtubei.js"; ++import { Counter, Registry } from "prom-client"; ++ ++export let metrics: Metrics | undefined; ++ ++export class Metrics { ++ private METRICS_PREFIX = "invidious_companion_"; ++ public register = new Registry(); ++ ++ public createCounter(name: string, help?: string): Counter { ++ return new Counter({ ++ name: `${this.METRICS_PREFIX}${name}`, ++ help: help || "No help has been provided for this metric", ++ registers: [this.register], ++ }); ++ } ++ ++ public potokenGenerationFailure = this.createCounter( ++ "potoken_generation_failure", ++ "Number of times that the PoToken generation job has failed for whatever reason", ++ ); ++ ++ private innertubeErrorStatusUnknown = this.createCounter( ++ "innertube_error_status_unknown", ++ "Number of times that an unknown status has been returned by Innertube API", ++ ); ++ ++ private innertubeErrorReasonSignIn = this.createCounter( ++ "innertube_error_reason_SignIn", ++ 'Number of times that the message "Sign in to confirm you’re not a bot." has been returned by Innertube API', ++ ); ++ ++ private innertubeErrorSubreasonProtectCommunity = this.createCounter( ++ "innertube_error_subreason_ProtectCommunity", ++ 'Number of times that the message "This helps protect our community." has been returned by Innertube API', ++ ); ++ ++ private innertubeErrorReasonUnknown = this.createCounter( ++ "innertube_error_reason_unknown", ++ "Number of times that an unknown reason has been returned by the Innertube API", ++ ); ++ ++ private innertubeErrorSubreasonUnknown = this.createCounter( ++ "innertube_error_subreason_unknown", ++ "Number of times that an unknown subreason has been returned by the Innertube API", ++ ); ++ ++ public innertubeSuccessfulRequest = this.createCounter( ++ "innertube_successful_request", ++ "Number successful requests made to the Innertube API", ++ ); ++ ++ private innertubeFailedRequest = this.createCounter( ++ "innertube_failed_request", ++ "Number failed requests made to the Innertube API for whatever reason", ++ ); ++ ++ public checkInnertubeResponse(videoData: IRawResponse) { ++ this.innertubeFailedRequest.inc(); ++ ++ switch (true) { ++ // CONTENT_CHECK_REQUIRED: Sensitive content videos. ++ case (videoData.playabilityStatus?.status === ++ "CONTENT_CHECK_REQUIRED"): { ++ break; ++ } ++ case (videoData.playabilityStatus?.status === "LOGIN_REQUIRED"): { ++ switch (true) { ++ // Age restricted videos, we don't need to track those. ++ case videoData.playabilityStatus?.reason?.includes( ++ "Sign in to confirm your age", ++ ): { ++ break; ++ } ++ ++ case videoData.playabilityStatus?.reason?.includes( ++ "Sign in to confirm you’re not a bot", ++ ): { ++ this.innertubeErrorReasonSignIn.inc(); ++ ++ switch (true) { ++ case videoData.playabilityStatus?.errorScreen ++ ?.playerErrorMessageRenderer ++ ?.subreason?.runs?.[0]?.text?.includes( ++ "This helps protect our community", ++ ): { ++ this.innertubeErrorSubreasonProtectCommunity ++ .inc(); ++ break; ++ } ++ default: { ++ this.innertubeErrorSubreasonUnknown.inc(); ++ break; ++ } ++ } ++ ++ break; ++ } ++ ++ default: { ++ this.innertubeErrorReasonUnknown.inc(); ++ break; ++ } ++ } ++ break; ++ } ++ default: ++ this.innertubeErrorStatusUnknown.inc(); ++ break; ++ } ++ } ++} +diff --git a/src/lib/helpers/youtubePlayerHandling.ts b/src/lib/helpers/youtubePlayerHandling.ts +index 396eabf..c099026 100644 +--- a/src/lib/helpers/youtubePlayerHandling.ts ++++ b/src/lib/helpers/youtubePlayerHandling.ts +@@ -2,6 +2,7 @@ import { ApiResponse, Innertube, YT } from "youtubei.js"; + import { generateRandomString } from "youtubei.js/Utils"; + import { compress, decompress } from "brotli"; + import type { BG } from "bgutils"; ++import { metrics } from "../helpers/metrics.ts"; + let youtubePlayerReqLocation = "youtubePlayerReq"; + if (Deno.env.get("YT_PLAYER_REQ_LOCATION")) { + if (Deno.env.has("DENO_COMPILED")) { +@@ -167,20 +168,25 @@ export const youtubePlayerParsing = async ({ + microformat, + }))(videoData); + +- if (cacheEnabled && videoData.playabilityStatus?.status == "OK") { +- (async () => { +- await kv.set( +- ["video_cache", videoId], +- compress( +- new TextEncoder().encode( +- JSON.stringify(videoOnlyNecessaryInfo), ++ if (videoData.playabilityStatus?.status == "OK") { ++ metrics?.innertubeSuccessfulRequest.inc(); ++ if (cacheEnabled) { ++ (async () => { ++ await kv.set( ++ ["video_cache", videoId], ++ compress( ++ new TextEncoder().encode( ++ JSON.stringify(videoOnlyNecessaryInfo), ++ ), + ), +- ), +- { +- expireIn: 1000 * 60 * 60, +- }, +- ); +- })(); ++ { ++ expireIn: 1000 * 60 * 60, ++ }, ++ ); ++ })(); ++ } ++ } else { ++ metrics?.checkInnertubeResponse(videoData); + } + + return videoOnlyNecessaryInfo; +diff --git a/src/lib/types/HonoVariables.ts b/src/lib/types/HonoVariables.ts +index 4f42d28..1e9b7a2 100644 +--- a/src/lib/types/HonoVariables.ts ++++ b/src/lib/types/HonoVariables.ts +@@ -1,9 +1,11 @@ + import { Innertube } from "youtubei.js"; + import { BG } from "bgutils"; + import type { Config } from "../helpers/config.ts"; ++import { Metrics } from "../helpers/metrics.ts"; + + export type HonoVariables = { + innertubeClient: Innertube; + config: Config; + tokenMinter: BG.WebPoMinter; ++ metrics: Metrics | undefined; + }; +diff --git a/src/main.ts b/src/main.ts +index b57cdc5..51ec965 100644 +--- a/src/main.ts ++++ b/src/main.ts +@@ -20,6 +20,7 @@ import type { HonoVariables } from "./lib/types/HonoVariables.ts"; + + import { parseConfig } from "./lib/helpers/config.ts"; + const config = await parseConfig(); ++import { Metrics } from "./lib/helpers/metrics.ts"; + + let getFetchClientLocation = "getFetchClient"; + if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) { +@@ -38,6 +39,7 @@ declare module "hono" { + interface ContextVariableMap extends HonoVariables {} + } + const app = new Hono(); ++const metrics = config.server.enable_metrics ? new Metrics() : undefined; + + let tokenMinter: BG.WebPoMinter; + let innertubeClient: Innertube; +@@ -170,11 +172,16 @@ if (!innertubeClientOauthEnabled) { + { backoffSchedule: [5_000, 15_000, 60_000, 180_000] }, + async () => { + if (innertubeClientJobPoTokenEnabled) { +- ({ innertubeClient, tokenMinter } = await poTokenGenerate( +- innertubeClient, +- config, +- innertubeClientCache, +- )); ++ try { ++ ({ innertubeClient, tokenMinter } = await poTokenGenerate( ++ innertubeClient, ++ config, ++ innertubeClientCache, ++ )); ++ } catch (err) { ++ metrics?.potokenGenerationFailure.inc(); ++ throw err; ++ } + } else { + innertubeClient = await Innertube.create({ + enable_session_cache: false, +@@ -212,6 +219,7 @@ app.use("*", async (c, next) => { + c.set("innertubeClient", innertubeClient); + c.set("tokenMinter", tokenMinter); + c.set("config", config); ++ c.set("metrics", metrics); + await next(); + }); + +diff --git a/src/routes/index.ts b/src/routes/index.ts +index 6448e3d..07ff900 100644 +--- a/src/routes/index.ts ++++ b/src/routes/index.ts +@@ -11,6 +11,7 @@ import videoPlaybackProxy from "./videoPlaybackProxy.ts"; + import health from "./health.ts"; + import type { Config } from "../lib/helpers/config.ts"; + import info from "./info.ts"; ++import metrics from "./metrics.ts"; + + export const routes = ( + app: Hono, +@@ -34,4 +35,7 @@ export const routes = ( + app.route("/videoplayback", videoPlaybackProxy); + app.route("/healthz", health); + app.route("/info", info); ++ if (config.server.enable_metrics) { ++ app.route("/metrics", metrics); ++ } + }; +diff --git a/src/routes/metrics.ts b/src/routes/metrics.ts +new file mode 100644 +index 0000000..8e0eea8 +--- /dev/null ++++ b/src/routes/metrics.ts +@@ -0,0 +1,11 @@ ++import { Hono } from "hono"; ++ ++const metrics = new Hono(); ++ ++metrics.get("/", async (c) => { ++ return new Response(await c.get("metrics")?.register.metrics(), { ++ headers: { "Content-Type": "text/plain" }, ++ }); ++}); ++ ++export default metrics; +diff --git a/src/routes/youtube_api_routes/player.ts b/src/routes/youtube_api_routes/player.ts +index 0b4ac0e..a4071cc 100644 +--- a/src/routes/youtube_api_routes/player.ts ++++ b/src/routes/youtube_api_routes/player.ts +@@ -7,6 +7,7 @@ player.post("/player", async (c) => { + const jsonReq = await c.req.json(); + const innertubeClient = c.get("innertubeClient"); + const config = c.get("config"); ++ const metrics = c.get("metrics"); + if (jsonReq.videoId) { + return c.json( + await youtubePlayerParsing({ +@@ -14,6 +15,7 @@ player.post("/player", async (c) => { + videoId: jsonReq.videoId, + config, + tokenMinter: c.get("tokenMinter"), ++ metrics, + }), + ); + } +-- +2.49.0 + diff --git a/patches/0013-add-metrics-for-proxy-retries.patch b/patches/0013-add-metrics-for-proxy-retries.patch new file mode 100644 index 0000000..b3e9009 --- /dev/null +++ b/patches/0013-add-metrics-for-proxy-retries.patch @@ -0,0 +1,41 @@ +From 25aecff44cfb65df0a20ff358e7a8371ea79c430 Mon Sep 17 00:00:00 2001 +From: Fijxu +Date: Tue, 25 Mar 2025 00:07:28 -0300 +Subject: [PATCH 13/13] add metrics for proxy retries + +--- + src/lib/helpers/metrics.ts | 5 +++++ + src/lib/helpers/youtubePlayerHandling.ts | 1 + + 2 files changed, 6 insertions(+) + +diff --git a/src/lib/helpers/metrics.ts b/src/lib/helpers/metrics.ts +index 98ca83d..93c6b82 100644 +--- a/src/lib/helpers/metrics.ts ++++ b/src/lib/helpers/metrics.ts +@@ -55,6 +55,11 @@ export class Metrics { + "Number failed requests made to the Innertube API for whatever reason", + ); + ++ public proxyRetry = this.createCounter( ++ "proxy_retry", ++ 'Times a request to innertube has been retried when it gets "This helps protect our community"', ++ ); ++ + public checkInnertubeResponse(videoData: IRawResponse) { + this.innertubeFailedRequest.inc(); + +diff --git a/src/lib/helpers/youtubePlayerHandling.ts b/src/lib/helpers/youtubePlayerHandling.ts +index c099026..2957e83 100644 +--- a/src/lib/helpers/youtubePlayerHandling.ts ++++ b/src/lib/helpers/youtubePlayerHandling.ts +@@ -60,6 +60,7 @@ export const youtubePlayerParsing = async ({ + console.log( + `[DEBUG] Got 'This helps protect our community', retrying request for ${videoId}. Retry ${retries} of ${maxRetries}`, + ); ++ metrics?.proxyRetry.inc(); + youtubePlayerResponse = await youtubePlayerReq( + innertubeClient, + videoId, +-- +2.49.0 +