From 9abed96033e64de18adaeccff66c1dbaeeaa04ef Mon Sep 17 00:00:00 2001 From: Fijxu Date: Wed, 9 Apr 2025 13:41:00 -0400 Subject: [PATCH] metrics: track unidentified innertube errors --- patches/0001-ci-update-deno-to-2.2.5.patch | 4 +- ...-for-an-external-videoplayback-proxy.patch | 4 +- ...external-videoplayback-proxy-via-inf.patch | 4 +- ...ion-limit-on-DASH-streams-to-save-ba.patch | 4 +- ...-env-variable-to-set-verify_requests.patch | 4 +- ...eat-add-support-for-multiple-proxies.patch | 4 +- ...-to-disable-potoken-generation-check.patch | 4 +- ...add-proxy-retries-on-innertube-error.patch | 4 +- .../0009-add-metrics-for-proxy-retries.patch | 4 +- ...10-fix-fix-tokio-overflow-on-compile.patch | 4 +- ...variable-for-youtube_session.frequen.patch | 4 +- .../0012-add-option-to-disable-logging.patch | 4 +- ...-track-unidentified-innertube-errors.patch | 214 ++++++++++++++++++ 13 files changed, 238 insertions(+), 24 deletions(-) create mode 100644 patches/0013-metrics-track-unidentified-innertube-errors.patch diff --git a/patches/0001-ci-update-deno-to-2.2.5.patch b/patches/0001-ci-update-deno-to-2.2.5.patch index 6e214ab..97e8ab0 100644 --- a/patches/0001-ci-update-deno-to-2.2.5.patch +++ b/patches/0001-ci-update-deno-to-2.2.5.patch @@ -1,7 +1,7 @@ -From 79366d2e76b2a7b5cf5285556d0ff9b6a87e305c Mon Sep 17 00:00:00 2001 +From c83645ec3e6d0a53ab20f0eb9e0b16e11f6d2fed Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 19:37:34 -0300 -Subject: [PATCH 01/12] ci: update deno to 2.2.5 +Subject: [PATCH 01/13] ci: update deno to 2.2.5 --- Dockerfile | 2 +- diff --git a/patches/0002-feat-add-support-for-an-external-videoplayback-proxy.patch b/patches/0002-feat-add-support-for-an-external-videoplayback-proxy.patch index 4c9c60d..aae374f 100644 --- a/patches/0002-feat-add-support-for-an-external-videoplayback-proxy.patch +++ b/patches/0002-feat-add-support-for-an-external-videoplayback-proxy.patch @@ -1,7 +1,7 @@ -From 30261e12366f3be7ac361e461feeeb2a0b269259 Mon Sep 17 00:00:00 2001 +From c87979c26380ed00e9894396a3d15d6c0d7059e4 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 18:44:10 -0300 -Subject: [PATCH 02/12] feat: add support for an external videoplayback proxy +Subject: [PATCH 02/13] feat: add support for an external videoplayback proxy --- config/config.example.toml | 1 + diff --git a/patches/0003-feat-report-the-external-videoplayback-proxy-via-inf.patch b/patches/0003-feat-report-the-external-videoplayback-proxy-via-inf.patch index b7b35ef..62c57f8 100644 --- a/patches/0003-feat-report-the-external-videoplayback-proxy-via-inf.patch +++ b/patches/0003-feat-report-the-external-videoplayback-proxy-via-inf.patch @@ -1,7 +1,7 @@ -From d1eb3b2bf7d999a9506b0705855e7f417636f2a8 Mon Sep 17 00:00:00 2001 +From 69060577293547a049f37e8319661d139bc0d8e0 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 18:52:53 -0300 -Subject: [PATCH 03/12] feat: report the external videoplayback proxy via /info +Subject: [PATCH 03/13] feat: report the external videoplayback proxy via /info endpoint --- diff --git a/patches/0004-feat-add-resolution-limit-on-DASH-streams-to-save-ba.patch b/patches/0004-feat-add-resolution-limit-on-DASH-streams-to-save-ba.patch index 2fc60c7..96131a4 100644 --- a/patches/0004-feat-add-resolution-limit-on-DASH-streams-to-save-ba.patch +++ b/patches/0004-feat-add-resolution-limit-on-DASH-streams-to-save-ba.patch @@ -1,7 +1,7 @@ -From 69b27055fd4cf52820416d4f06b9e68952794a6d Mon Sep 17 00:00:00 2001 +From 84db2add0f0c4e3ab299d999e85966029f8d3c53 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 19:02:01 -0300 -Subject: [PATCH 04/12] feat: add resolution limit on DASH streams to save +Subject: [PATCH 04/13] feat: add resolution limit on DASH streams to save bandwidth --- diff --git a/patches/0005-feat-add-env-variable-to-set-verify_requests.patch b/patches/0005-feat-add-env-variable-to-set-verify_requests.patch index 520601d..e54b886 100644 --- a/patches/0005-feat-add-env-variable-to-set-verify_requests.patch +++ b/patches/0005-feat-add-env-variable-to-set-verify_requests.patch @@ -1,7 +1,7 @@ -From 2c330e48e3a97ce12c6b3498e97abbb53493dd1e Mon Sep 17 00:00:00 2001 +From bba074b050e57a1c59e97a966568eb98de192606 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 19:06:04 -0300 -Subject: [PATCH 05/12] feat: add env variable to set verify_requests +Subject: [PATCH 05/13] feat: add env variable to set verify_requests --- src/lib/helpers/config.ts | 4 +++- diff --git a/patches/0006-feat-add-support-for-multiple-proxies.patch b/patches/0006-feat-add-support-for-multiple-proxies.patch index ed4c815..67e3035 100644 --- a/patches/0006-feat-add-support-for-multiple-proxies.patch +++ b/patches/0006-feat-add-support-for-multiple-proxies.patch @@ -1,7 +1,7 @@ -From 13fd8adcfc0a4a5f02dca24e8dfa7f89412c8c8e Mon Sep 17 00:00:00 2001 +From e2828064cea085c5994080cf73e162a62f86403f Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 19:20:52 -0300 -Subject: [PATCH 06/12] feat: add support for multiple proxies +Subject: [PATCH 06/13] feat: add support for multiple proxies --- src/lib/helpers/getFetchClient.ts | 17 ++++++++++++++++- diff --git a/patches/0007-feat-add-option-to-disable-potoken-generation-check.patch b/patches/0007-feat-add-option-to-disable-potoken-generation-check.patch index 30dbf2a..2c6f1a9 100644 --- a/patches/0007-feat-add-option-to-disable-potoken-generation-check.patch +++ b/patches/0007-feat-add-option-to-disable-potoken-generation-check.patch @@ -1,7 +1,7 @@ -From 849be9e4b94bd415140f4dfc5223598234b9e83f Mon Sep 17 00:00:00 2001 +From 96220305843a6fbfb99387b0adcc1a7e5d2a3d6f Mon Sep 17 00:00:00 2001 From: Fijxu Date: Mon, 24 Mar 2025 20:34:33 -0300 -Subject: [PATCH 07/12] feat: add option to disable potoken generation check +Subject: [PATCH 07/13] feat: add option to disable potoken generation check --- config/config.example.toml | 1 + diff --git a/patches/0008-add-proxy-retries-on-innertube-error.patch b/patches/0008-add-proxy-retries-on-innertube-error.patch index 4ea47e7..a5c1a26 100644 --- a/patches/0008-add-proxy-retries-on-innertube-error.patch +++ b/patches/0008-add-proxy-retries-on-innertube-error.patch @@ -1,7 +1,7 @@ -From 20fe722f29891d047ba966b4d98e445a3620037e Mon Sep 17 00:00:00 2001 +From 8440076ee1af7a3088938a9e481afbaede9b5750 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Tue, 25 Mar 2025 00:04:47 -0300 -Subject: [PATCH 08/12] add proxy retries on innertube error +Subject: [PATCH 08/13] add proxy retries on innertube error --- src/lib/helpers/config.ts | 1 + diff --git a/patches/0009-add-metrics-for-proxy-retries.patch b/patches/0009-add-metrics-for-proxy-retries.patch index 3c14679..5c5dca8 100644 --- a/patches/0009-add-metrics-for-proxy-retries.patch +++ b/patches/0009-add-metrics-for-proxy-retries.patch @@ -1,7 +1,7 @@ -From e7834baf2a05cbc724263f3edde2f9f27d900e0c Mon Sep 17 00:00:00 2001 +From 79ef8ba531499997aea3e74415a2e8c315080e59 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Tue, 25 Mar 2025 00:07:28 -0300 -Subject: [PATCH 09/12] add metrics for proxy retries +Subject: [PATCH 09/13] add metrics for proxy retries --- src/lib/helpers/metrics.ts | 5 +++++ diff --git a/patches/0010-fix-fix-tokio-overflow-on-compile.patch b/patches/0010-fix-fix-tokio-overflow-on-compile.patch index a26fcc7..164ceda 100644 --- a/patches/0010-fix-fix-tokio-overflow-on-compile.patch +++ b/patches/0010-fix-fix-tokio-overflow-on-compile.patch @@ -1,7 +1,7 @@ -From 64b034eb8b365d2f957ffb4ec1975e5744197269 Mon Sep 17 00:00:00 2001 +From 34010c0db0dbbd9f0831eb44ed22ff728e7fe0e1 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Tue, 25 Mar 2025 00:24:07 -0300 -Subject: [PATCH 10/12] fix: fix tokio overflow on compile +Subject: [PATCH 10/13] fix: fix tokio overflow on compile --- Dockerfile | 2 ++ diff --git a/patches/0011-Add-environment-variable-for-youtube_session.frequen.patch b/patches/0011-Add-environment-variable-for-youtube_session.frequen.patch index 42d1ece..2100546 100644 --- a/patches/0011-Add-environment-variable-for-youtube_session.frequen.patch +++ b/patches/0011-Add-environment-variable-for-youtube_session.frequen.patch @@ -1,7 +1,7 @@ -From 2841bfba51d147c5dd8061225b13d34a81398d00 Mon Sep 17 00:00:00 2001 +From 9bfe190249affbdb92ecfa7bb575a2948682f836 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Wed, 26 Mar 2025 12:24:49 -0300 -Subject: [PATCH 11/12] Add environment variable for youtube_session.frequency +Subject: [PATCH 11/13] Add environment variable for youtube_session.frequency --- src/lib/helpers/config.ts | 4 +++- diff --git a/patches/0012-add-option-to-disable-logging.patch b/patches/0012-add-option-to-disable-logging.patch index ce8bea0..1341f7a 100644 --- a/patches/0012-add-option-to-disable-logging.patch +++ b/patches/0012-add-option-to-disable-logging.patch @@ -1,7 +1,7 @@ -From 850288ff13a962271a676eea38b526ddf9fcba0f Mon Sep 17 00:00:00 2001 +From 0916b71023628157a597339e4a0481c5dec1315d Mon Sep 17 00:00:00 2001 From: Fijxu Date: Tue, 8 Apr 2025 02:21:10 -0400 -Subject: [PATCH 12/12] add option to disable logging +Subject: [PATCH 12/13] add option to disable logging --- src/lib/helpers/config.ts | 7 ++++++- diff --git a/patches/0013-metrics-track-unidentified-innertube-errors.patch b/patches/0013-metrics-track-unidentified-innertube-errors.patch new file mode 100644 index 0000000..ecc7dfb --- /dev/null +++ b/patches/0013-metrics-track-unidentified-innertube-errors.patch @@ -0,0 +1,214 @@ +From e3cfa0f2dc3f7fc74c90db9f5636e7b4c43eb0c8 Mon Sep 17 00:00:00 2001 +From: Fijxu +Date: Wed, 9 Apr 2025 13:37:16 -0400 +Subject: [PATCH 13/13] metrics: track unidentified innertube errors + +--- + src/lib/helpers/config.ts | 4 ++ + src/lib/helpers/metrics.ts | 117 +++++++++++++++++++++++++++++++------ + src/main.ts | 4 +- + 3 files changed, 106 insertions(+), 19 deletions(-) + +diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts +index acb04bb..20e5560 100644 +--- a/src/lib/helpers/config.ts ++++ b/src/lib/helpers/config.ts +@@ -23,6 +23,10 @@ export const ConfigSchema = z.object({ + disable_logs: z.boolean().default( + Deno.env.get("SERVER_DISABLE_LOGS") === "true" || false, + ), ++ track_unknown_innertube_errors: z.boolean().default( ++ Deno.env.get("SERVER_TRACK_UNKNOWN_INNERTUBE_ERRORS") === "true" || ++ false, ++ ), + }).strict().default({}), + cache: z.object({ + enabled: z.boolean().default(true), +diff --git a/src/lib/helpers/metrics.ts b/src/lib/helpers/metrics.ts +index 5dee540..c22c792 100644 +--- a/src/lib/helpers/metrics.ts ++++ b/src/lib/helpers/metrics.ts +@@ -2,14 +2,24 @@ import { IRawResponse } from "youtubei.js"; + import { Counter, Registry } from "prom-client"; + + export class Metrics { ++ trackUnknownInnertubeErrors: boolean; + private METRICS_PREFIX = "invidious_companion_"; + public register = new Registry(); + +- public createCounter(name: string, help?: string): Counter { ++ constructor(trackUnknownInnertubeErrors: boolean) { ++ this.trackUnknownInnertubeErrors = trackUnknownInnertubeErrors; ++ } ++ ++ public createCounter( ++ name: string, ++ help?: string, ++ labels?: string[], ++ ): Counter { + return new Counter({ + name: `${this.METRICS_PREFIX}${name}`, + help: help || "No help has been provided for this metric", + registers: [this.register], ++ labelNames: Array.isArray(labels) ? labels : [], + }); + } + +@@ -26,6 +36,7 @@ export class Metrics { + private innertubeErrorStatusUnknown = this.createCounter( + "innertube_error_status_unknown_total", + "Number of times that an unknown status has been returned by Innertube API", ++ ["error"], + ); + + private innertubeErrorReasonSignIn = this.createCounter( +@@ -66,26 +77,62 @@ export class Metrics { + private checkStatus(videoData: IRawResponse) { + const status = videoData.playabilityStatus?.status; + +- return { +- unplayable: status === +- "UNPLAYABLE", +- contentCheckRequired: status === +- "CONTENT_CHECK_REQUIRED", +- loginRequired: status === "LOGIN_REQUIRED", ++ interface Error { ++ unplayable: boolean; ++ contentCheckRequired: boolean; ++ loginRequired: boolean; ++ unknown: string | undefined; ++ } ++ ++ const error: Error = { ++ unplayable: false, ++ contentCheckRequired: false, ++ loginRequired: false, ++ unknown: undefined, + }; ++ ++ switch (status) { ++ case "UNPLAYABLE": ++ error.unplayable = true; ++ return error; ++ case "CONTENT_CHECK_REQUIRED": ++ error.contentCheckRequired = true; ++ return error; ++ case "LOGIN_REQUIRED": ++ error.loginRequired = true; ++ return error; ++ default: ++ error.unknown = status; ++ return error; ++ } + } + + private checkReason(videoData: IRawResponse) { + const reason = videoData.playabilityStatus?.reason; + +- return { +- signInToConfirmAge: reason?.includes( +- "Sign in to confirm your age", +- ), +- SignInToConfirmBot: reason?.includes( +- "Sign in to confirm you’re not a bot", +- ), ++ interface Error { ++ signInToConfirmAge: boolean; ++ SignInToConfirmBot: boolean; ++ unknown: string | undefined; ++ } ++ ++ const error: Error = { ++ signInToConfirmAge: false, ++ SignInToConfirmBot: false, ++ unknown: undefined, + }; ++ ++ switch (true) { ++ case reason?.includes("Sign in to confirm your age"): ++ error.signInToConfirmAge = true; ++ return error; ++ case reason?.includes("Sign in to confirm you’re not a bot"): ++ error.SignInToConfirmBot = true; ++ return error; ++ default: ++ error.unknown = reason; ++ return error; ++ } + } + + private checkSubreason(videoData: IRawResponse) { +@@ -93,17 +140,51 @@ export class Metrics { + ?.playerErrorMessageRenderer + ?.subreason?.runs?.[0]?.text; + +- return { +- thisHelpsProtectCommunity: subReason?.includes( +- "This helps protect our community", +- ), ++ interface Error { ++ thisHelpsProtectCommunity: boolean; ++ unknown: string | undefined; ++ } ++ ++ const error: Error = { ++ thisHelpsProtectCommunity: false, ++ unknown: undefined, + }; ++ ++ switch (true) { ++ case subReason?.includes("This helps protect our community"): ++ error.thisHelpsProtectCommunity = true; ++ return error; ++ default: ++ error.unknown = subReason; ++ return error; ++ } + } + + public checkInnertubeResponse(videoData: IRawResponse) { + this.innertubeFailedRequest.inc(); + const status = this.checkStatus(videoData); + ++ if (this.trackUnknownInnertubeErrors) { ++ if (status?.unknown) { ++ this.innertubeErrorStatusUnknown.labels({ ++ error: status.unknown, ++ }).inc(); ++ const reason = this.checkReason(videoData); ++ if (reason) { ++ this.innertubeErrorReasonUnknown.labels({ ++ error: status.unknown, ++ }).inc(); ++ const subReason = this.checkSubreason(videoData); ++ if (subReason) { ++ this.innertubeErrorStatusUnknown.labels({ ++ error: status.unknown, ++ }).inc(); ++ } ++ } ++ return; ++ } ++ } ++ + if (status.contentCheckRequired || status.unplayable) return; + + if (status.loginRequired) { +diff --git a/src/main.ts b/src/main.ts +index f659d97..84ef286 100644 +--- a/src/main.ts ++++ b/src/main.ts +@@ -27,7 +27,9 @@ declare module "hono" { + interface ContextVariableMap extends HonoVariables {} + } + const app = new Hono(); +-const metrics = config.server.enable_metrics ? new Metrics() : undefined; ++const metrics = config.server.enable_metrics ++ ? new Metrics(config.server.track_unknown_innertube_errors) ++ : undefined; + + let tokenMinter: TokenMinter; + let innertubeClient: Innertube; +-- +2.49.0 +