From 5352eed8c5ba428ac3e17cec5105e0c4e23db53b Mon Sep 17 00:00:00 2001 From: Fijxu Date: Tue, 18 Mar 2025 16:38:23 -0300 Subject: [PATCH 12/14] add support for prometheus metrics fix deno lint and typo chore: fmt apply suggestions better innertube response checking rename route _metrics to metrics use Hono context to pass Metrics onto functions fix: add missing metrics argument --- config/config.example.toml | 1 + deno.json | 1 + deno.lock | 25 ++++- src/lib/helpers/config.ts | 11 +- src/lib/helpers/metrics.ts | 112 +++++++++++++++++++ src/lib/helpers/youtubePlayerHandling.ts | 34 +++--- src/lib/jobs/potoken.ts | 3 + src/lib/types/HonoVariables.ts | 2 + src/main.ts | 20 +++- src/routes/index.ts | 4 + src/routes/invidious_routes/captions.ts | 2 + src/routes/invidious_routes/dashManifest.ts | 2 + src/routes/invidious_routes/latestVersion.ts | 2 + src/routes/metrics.ts | 11 ++ src/routes/youtube_api_routes/player.ts | 2 + 15 files changed, 211 insertions(+), 21 deletions(-) create mode 100644 src/lib/helpers/metrics.ts create mode 100644 src/routes/metrics.ts diff --git a/config/config.example.toml b/config/config.example.toml index 4c04ee3..01c3f83 100644 --- a/config/config.example.toml +++ b/config/config.example.toml @@ -16,6 +16,7 @@ # secret_key = "CHANGE_ME" # env variable: SERVER_SECRET_KEY # verify_requests = false # max_dash_resolution = 1080 +# enable_metrics = false # env variable: ENABLE_METRICS # [cache] # enabled = true diff --git a/deno.json b/deno.json index 02ea3a2..97600ec 100644 --- a/deno.json +++ b/deno.json @@ -6,6 +6,7 @@ "imports": { "hono": "jsr:@hono/hono@4.7.4", "@std/toml": "jsr:@std/toml@1.0.2", + "prom-client": "npm:prom-client@15.1.3", "youtubei.js": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno.ts", "youtubei.js/Utils": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/utils/Utils.ts", "youtubei.js/NavigationEndpoint": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/parser/classes/NavigationEndpoint.ts", diff --git a/deno.lock b/deno.lock index 27d8ae6..a9c62e7 100644 --- a/deno.lock +++ b/deno.lock @@ -15,7 +15,8 @@ "npm:@bufbuild/protobuf@2": "2.2.4", "npm:@types/estree@^1.0.6": "1.0.6", "npm:acorn@^8.8.0": "8.14.1", - "npm:jsdom@26.0.0": "26.0.0" + "npm:jsdom@26.0.0": "26.0.0", + "npm:prom-client@15.1.3": "15.1.3" }, "jsr": { "@hono/hono@4.7.4": { @@ -101,6 +102,9 @@ "@csstools/css-tokenizer@3.0.3": { "integrity": "sha512-UJnjoFsmxfKUdNYdWgOB0mWUypuLvAfQPH1+pyvRJs6euowbFkFC6P13w1l8mJyi3vxYMxc9kld5jZEGRQs6bw==" }, + "@opentelemetry/api@1.9.0": { + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==" + }, "@types/estree@1.0.6": { "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==" }, @@ -113,6 +117,9 @@ "asynckit@0.4.0": { "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, + "bintrees@1.0.2": { + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" + }, "call-bind-apply-helpers@1.0.2": { "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", "dependencies": [ @@ -318,6 +325,13 @@ "entities" ] }, + "prom-client@15.1.3": { + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "dependencies": [ + "@opentelemetry/api", + "tdigest" + ] + }, "punycode@2.3.1": { "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==" }, @@ -336,6 +350,12 @@ "symbol-tree@3.2.4": { "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" }, + "tdigest@0.1.2": { + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "dependencies": [ + "bintrees" + ] + }, "tldts-core@6.1.84": { "integrity": "sha512-NaQa1W76W2aCGjXybvnMYzGSM4x8fvG2AN/pla7qxcg0ZHbooOPhA8kctmOZUDfZyhDL27OGNbwAeig8P4p1vg==" }, @@ -1102,7 +1122,8 @@ "jsr:@std/fs@1.0.14", "jsr:@std/path@1.0.8", "jsr:@std/toml@1.0.2", - "npm:jsdom@26.0.0" + "npm:jsdom@26.0.0", + "npm:prom-client@15.1.3" ] } } diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts index d539738..646132f 100644 --- a/src/lib/helpers/config.ts +++ b/src/lib/helpers/config.ts @@ -23,6 +23,13 @@ const ConfigSchema = z.object({ ? false : true, ), + enable_metrics: z.boolean().default( + Deno.env.get("ENABLE_METRICS") === "true" + ? true + : Deno.env.get("ENABLE_METRICS") === "false" + ? false + : true, + ), }).strict().default({}), cache: z.object({ enabled: z.boolean().default(true), @@ -43,7 +50,9 @@ const ConfigSchema = z.object({ external_videoplayback_proxy: z.string().default( Deno.env.get("EXTERNAL_VIDEOPLAYBACK_PROXY") || "", ), - max_proxy_retries: z.number().default(Number(Deno.env.get("MAX_PROXY_RETIRES") || 5)), + max_proxy_retries: z.number().default( + Number(Deno.env.get("MAX_PROXY_RETIRES") || 5), + ), }).strict().default({}), jobs: z.object({ youtube_session: z.object({ diff --git a/src/lib/helpers/metrics.ts b/src/lib/helpers/metrics.ts new file mode 100644 index 0000000..98ca83d --- /dev/null +++ b/src/lib/helpers/metrics.ts @@ -0,0 +1,112 @@ +import { IRawResponse } from "youtubei.js"; +import { Counter, Registry } from "prom-client"; + +export let metrics: Metrics | undefined; + +export class Metrics { + private METRICS_PREFIX = "invidious_companion_"; + public register = new Registry(); + + public createCounter(name: string, help?: string): Counter { + return new Counter({ + name: `${this.METRICS_PREFIX}${name}`, + help: help || "No help has been provided for this metric", + registers: [this.register], + }); + } + + public potokenGenerationFailure = this.createCounter( + "potoken_generation_failure", + "Number of times that the PoToken generation job has failed for whatever reason", + ); + + private innertubeErrorStatusUnknown = this.createCounter( + "innertube_error_status_unknown", + "Number of times that an unknown status has been returned by Innertube API", + ); + + private innertubeErrorReasonSignIn = this.createCounter( + "innertube_error_reason_SignIn", + 'Number of times that the message "Sign in to confirm you’re not a bot." has been returned by Innertube API', + ); + + private innertubeErrorSubreasonProtectCommunity = this.createCounter( + "innertube_error_subreason_ProtectCommunity", + 'Number of times that the message "This helps protect our community." has been returned by Innertube API', + ); + + private innertubeErrorReasonUnknown = this.createCounter( + "innertube_error_reason_unknown", + "Number of times that an unknown reason has been returned by the Innertube API", + ); + + private innertubeErrorSubreasonUnknown = this.createCounter( + "innertube_error_subreason_unknown", + "Number of times that an unknown subreason has been returned by the Innertube API", + ); + + public innertubeSuccessfulRequest = this.createCounter( + "innertube_successful_request", + "Number successful requests made to the Innertube API", + ); + + private innertubeFailedRequest = this.createCounter( + "innertube_failed_request", + "Number failed requests made to the Innertube API for whatever reason", + ); + + public checkInnertubeResponse(videoData: IRawResponse) { + this.innertubeFailedRequest.inc(); + + switch (true) { + // CONTENT_CHECK_REQUIRED: Sensitive content videos. + case (videoData.playabilityStatus?.status === + "CONTENT_CHECK_REQUIRED"): { + break; + } + case (videoData.playabilityStatus?.status === "LOGIN_REQUIRED"): { + switch (true) { + // Age restricted videos, we don't need to track those. + case videoData.playabilityStatus?.reason?.includes( + "Sign in to confirm your age", + ): { + break; + } + + case videoData.playabilityStatus?.reason?.includes( + "Sign in to confirm you’re not a bot", + ): { + this.innertubeErrorReasonSignIn.inc(); + + switch (true) { + case videoData.playabilityStatus?.errorScreen + ?.playerErrorMessageRenderer + ?.subreason?.runs?.[0]?.text?.includes( + "This helps protect our community", + ): { + this.innertubeErrorSubreasonProtectCommunity + .inc(); + break; + } + default: { + this.innertubeErrorSubreasonUnknown.inc(); + break; + } + } + + break; + } + + default: { + this.innertubeErrorReasonUnknown.inc(); + break; + } + } + break; + } + default: + this.innertubeErrorStatusUnknown.inc(); + break; + } + } +} diff --git a/src/lib/helpers/youtubePlayerHandling.ts b/src/lib/helpers/youtubePlayerHandling.ts index 396eabf..2ae878d 100644 --- a/src/lib/helpers/youtubePlayerHandling.ts +++ b/src/lib/helpers/youtubePlayerHandling.ts @@ -2,6 +2,7 @@ import { ApiResponse, Innertube, YT } from "youtubei.js"; import { generateRandomString } from "youtubei.js/Utils"; import { compress, decompress } from "brotli"; import type { BG } from "bgutils"; +import { Metrics } from "../helpers/metrics.ts"; let youtubePlayerReqLocation = "youtubePlayerReq"; if (Deno.env.get("YT_PLAYER_REQ_LOCATION")) { if (Deno.env.has("DENO_COMPILED")) { @@ -24,12 +25,14 @@ export const youtubePlayerParsing = async ({ videoId, config, tokenMinter, + metrics, overrideCache = false, }: { innertubeClient: Innertube; videoId: string; config: Config; tokenMinter: BG.WebPoMinter; + metrics: Metrics | undefined; overrideCache?: boolean; }): Promise => { const cacheEnabled = overrideCache ? false : config.cache.enabled; @@ -167,20 +170,25 @@ export const youtubePlayerParsing = async ({ microformat, }))(videoData); - if (cacheEnabled && videoData.playabilityStatus?.status == "OK") { - (async () => { - await kv.set( - ["video_cache", videoId], - compress( - new TextEncoder().encode( - JSON.stringify(videoOnlyNecessaryInfo), + if (videoData.playabilityStatus?.status == "OK") { + metrics?.innertubeSuccessfulRequest.inc(); + if (cacheEnabled) { + (async () => { + await kv.set( + ["video_cache", videoId], + compress( + new TextEncoder().encode( + JSON.stringify(videoOnlyNecessaryInfo), + ), ), - ), - { - expireIn: 1000 * 60 * 60, - }, - ); - })(); + { + expireIn: 1000 * 60 * 60, + }, + ); + })(); + } + } else { + metrics?.checkInnertubeResponse(videoData); } return videoOnlyNecessaryInfo; diff --git a/src/lib/jobs/potoken.ts b/src/lib/jobs/potoken.ts index c7994c5..2acc957 100644 --- a/src/lib/jobs/potoken.ts +++ b/src/lib/jobs/potoken.ts @@ -8,6 +8,7 @@ import { youtubeVideoInfo, } from "../helpers/youtubePlayerHandling.ts"; import type { Config } from "../helpers/config.ts"; +import { Metrics } from "../helpers/metrics.ts"; let getFetchClientLocation = "getFetchClient"; if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) { if (Deno.env.has("DENO_COMPILED")) { @@ -26,6 +27,7 @@ export const poTokenGenerate = async ( innertubeClient: Innertube, config: Config, innertubeClientCache: UniversalCache, + metrics: Metrics | undefined, ): Promise<{ innertubeClient: Innertube; tokenMinter: BG.WebPoMinter }> => { if (innertubeClient.session.po_token) { innertubeClient = await Innertube.create({ @@ -160,6 +162,7 @@ export const poTokenGenerate = async ( videoId: video.id, config, tokenMinter: integrityTokenBasedMinter, + metrics, overrideCache: true, }); const videoInfo = youtubeVideoInfo( diff --git a/src/lib/types/HonoVariables.ts b/src/lib/types/HonoVariables.ts index 4f42d28..1e9b7a2 100644 --- a/src/lib/types/HonoVariables.ts +++ b/src/lib/types/HonoVariables.ts @@ -1,9 +1,11 @@ import { Innertube } from "youtubei.js"; import { BG } from "bgutils"; import type { Config } from "../helpers/config.ts"; +import { Metrics } from "../helpers/metrics.ts"; export type HonoVariables = { innertubeClient: Innertube; config: Config; tokenMinter: BG.WebPoMinter; + metrics: Metrics | undefined; }; diff --git a/src/main.ts b/src/main.ts index b57cdc5..952b9f8 100644 --- a/src/main.ts +++ b/src/main.ts @@ -20,6 +20,7 @@ import type { HonoVariables } from "./lib/types/HonoVariables.ts"; import { parseConfig } from "./lib/helpers/config.ts"; const config = await parseConfig(); +import { Metrics } from "./lib/helpers/metrics.ts"; let getFetchClientLocation = "getFetchClient"; if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) { @@ -38,6 +39,7 @@ declare module "hono" { interface ContextVariableMap extends HonoVariables {} } const app = new Hono(); +const metrics = config.server.enable_metrics ? new Metrics() : undefined; let tokenMinter: BG.WebPoMinter; let innertubeClient: Innertube; @@ -160,6 +162,7 @@ if (!innertubeClientOauthEnabled) { innertubeClient, config, innertubeClientCache as UniversalCache, + metrics, ), { minTimeout: 1_000, maxTimeout: 60_000, multiplier: 5, jitter: 0 }, )); @@ -170,11 +173,17 @@ if (!innertubeClientOauthEnabled) { { backoffSchedule: [5_000, 15_000, 60_000, 180_000] }, async () => { if (innertubeClientJobPoTokenEnabled) { - ({ innertubeClient, tokenMinter } = await poTokenGenerate( - innertubeClient, - config, - innertubeClientCache, - )); + try { + ({ innertubeClient, tokenMinter } = await poTokenGenerate( + innertubeClient, + config, + innertubeClientCache, + metrics, + )); + } catch (err) { + metrics?.potokenGenerationFailure.inc(); + throw err; + } } else { innertubeClient = await Innertube.create({ enable_session_cache: false, @@ -212,6 +221,7 @@ app.use("*", async (c, next) => { c.set("innertubeClient", innertubeClient); c.set("tokenMinter", tokenMinter); c.set("config", config); + c.set("metrics", metrics); await next(); }); diff --git a/src/routes/index.ts b/src/routes/index.ts index 6448e3d..07ff900 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -11,6 +11,7 @@ import videoPlaybackProxy from "./videoPlaybackProxy.ts"; import health from "./health.ts"; import type { Config } from "../lib/helpers/config.ts"; import info from "./info.ts"; +import metrics from "./metrics.ts"; export const routes = ( app: Hono, @@ -34,4 +35,7 @@ export const routes = ( app.route("/videoplayback", videoPlaybackProxy); app.route("/healthz", health); app.route("/info", info); + if (config.server.enable_metrics) { + app.route("/metrics", metrics); + } }; diff --git a/src/routes/invidious_routes/captions.ts b/src/routes/invidious_routes/captions.ts index 1eaeb14..f900298 100644 --- a/src/routes/invidious_routes/captions.ts +++ b/src/routes/invidious_routes/captions.ts @@ -19,6 +19,7 @@ const captionsHandler = new Hono<{ Variables: HonoVariables }>(); captionsHandler.get("/:videoId", async (c) => { const { videoId } = c.req.param(); const config = c.get("config"); + const metrics = c.get("metrics"); const check = c.req.query("check"); @@ -41,6 +42,7 @@ captionsHandler.get("/:videoId", async (c) => { videoId, config, tokenMinter: c.get("tokenMinter"), + metrics, }); const videoInfo = youtubeVideoInfo( diff --git a/src/routes/invidious_routes/dashManifest.ts b/src/routes/invidious_routes/dashManifest.ts index a4c0950..414b4cf 100644 --- a/src/routes/invidious_routes/dashManifest.ts +++ b/src/routes/invidious_routes/dashManifest.ts @@ -17,6 +17,7 @@ dashManifest.get("/:videoId", async (c) => { const innertubeClient = c.get("innertubeClient"); const config = c.get("config"); + const metrics = c.get("metrics"); if (config.server.verify_requests && check == undefined) { throw new HTTPException(400, { @@ -35,6 +36,7 @@ dashManifest.get("/:videoId", async (c) => { videoId, config, tokenMinter: c.get("tokenMinter"), + metrics, }); const videoInfo = youtubeVideoInfo( innertubeClient, diff --git a/src/routes/invidious_routes/latestVersion.ts b/src/routes/invidious_routes/latestVersion.ts index f1a7605..dbf57c2 100644 --- a/src/routes/invidious_routes/latestVersion.ts +++ b/src/routes/invidious_routes/latestVersion.ts @@ -21,6 +21,7 @@ latestVersion.get("/", async (c) => { const innertubeClient = c.get("innertubeClient"); const config = c.get("config"); + const metrics = c.get("metrics"); if (config.server.verify_requests && check == undefined) { throw new HTTPException(400, { @@ -39,6 +40,7 @@ latestVersion.get("/", async (c) => { videoId: id, config, tokenMinter: c.get("tokenMinter"), + metrics, }); const videoInfo = youtubeVideoInfo( innertubeClient, diff --git a/src/routes/metrics.ts b/src/routes/metrics.ts new file mode 100644 index 0000000..8e0eea8 --- /dev/null +++ b/src/routes/metrics.ts @@ -0,0 +1,11 @@ +import { Hono } from "hono"; + +const metrics = new Hono(); + +metrics.get("/", async (c) => { + return new Response(await c.get("metrics")?.register.metrics(), { + headers: { "Content-Type": "text/plain" }, + }); +}); + +export default metrics; diff --git a/src/routes/youtube_api_routes/player.ts b/src/routes/youtube_api_routes/player.ts index 0b4ac0e..a4071cc 100644 --- a/src/routes/youtube_api_routes/player.ts +++ b/src/routes/youtube_api_routes/player.ts @@ -7,6 +7,7 @@ player.post("/player", async (c) => { const jsonReq = await c.req.json(); const innertubeClient = c.get("innertubeClient"); const config = c.get("config"); + const metrics = c.get("metrics"); if (jsonReq.videoId) { return c.json( await youtubePlayerParsing({ @@ -14,6 +15,7 @@ player.post("/player", async (c) => { videoId: jsonReq.videoId, config, tokenMinter: c.get("tokenMinter"), + metrics, }), ); } -- 2.49.0