Fijxu/invidious-companion-patches
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions
invidious-companion-patches/patches/0012-add-support-for-prometheus-metrics.patch
Fijxu dd81e6b4f1
Some checks failed
Build and Push Docker Image / build-and-push (push) Failing after 42s
Details
add more patches xd
2025-03-25 00:11:23 -03:00

453 lines
16 KiB
Diff
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

From af90ccd9de22acefec3a7a035a6a2e4e7fc309fb Mon Sep 17 00:00:00 2001
From: Fijxu <fijxu@nadeko.net>
Date: Tue, 18 Mar 2025 16:38:23 -0300
Subject: [PATCH 12/13] add support for prometheus metrics
fix deno lint and typo
chore: fmt
apply suggestions
better innertube response checking
rename route _metrics to metrics
use Hono context to pass Metrics onto functions
fix: add missing metrics argument
---
config/config.example.toml | 1 +
deno.json | 1 +
deno.lock | 25 ++++-
src/lib/helpers/config.ts | 11 ++-
src/lib/helpers/metrics.ts | 112 +++++++++++++++++++++++
src/lib/helpers/youtubePlayerHandling.ts | 32 ++++---
src/lib/types/HonoVariables.ts | 2 +
src/main.ts | 18 +++-
src/routes/index.ts | 4 +
src/routes/metrics.ts | 11 +++
src/routes/youtube_api_routes/player.ts | 2 +
11 files changed, 198 insertions(+), 21 deletions(-)
create mode 100644 src/lib/helpers/metrics.ts
create mode 100644 src/routes/metrics.ts
diff --git a/config/config.example.toml b/config/config.example.toml
index 4c04ee3..01c3f83 100644
--- a/config/config.example.toml
+++ b/config/config.example.toml
@@ -16,6 +16,7 @@
# secret_key = "CHANGE_ME" # env variable: SERVER_SECRET_KEY
# verify_requests = false
# max_dash_resolution = 1080
+# enable_metrics = false # env variable: ENABLE_METRICS
# [cache]
# enabled = true
diff --git a/deno.json b/deno.json
index 02ea3a2..97600ec 100644
--- a/deno.json
+++ b/deno.json
@@ -6,6 +6,7 @@
"imports": {
"hono": "jsr:@hono/hono@4.7.4",
"@std/toml": "jsr:@std/toml@1.0.2",
+ "prom-client": "npm:prom-client@15.1.3",
"youtubei.js": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno.ts",
"youtubei.js/Utils": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/utils/Utils.ts",
"youtubei.js/NavigationEndpoint": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/parser/classes/NavigationEndpoint.ts",
diff --git a/deno.lock b/deno.lock
index 27d8ae6..a9c62e7 100644
--- a/deno.lock
+++ b/deno.lock
@@ -15,7 +15,8 @@
"npm:@bufbuild/protobuf@2": "2.2.4",
"npm:@types/estree@^1.0.6": "1.0.6",
"npm:acorn@^8.8.0": "8.14.1",
- "npm:jsdom@26.0.0": "26.0.0"
+ "npm:jsdom@26.0.0": "26.0.0",
+ "npm:prom-client@15.1.3": "15.1.3"
},
"jsr": {
"@hono/hono@4.7.4": {
@@ -101,6 +102,9 @@
"@csstools/css-tokenizer@3.0.3": {
"integrity": "sha512-UJnjoFsmxfKUdNYdWgOB0mWUypuLvAfQPH1+pyvRJs6euowbFkFC6P13w1l8mJyi3vxYMxc9kld5jZEGRQs6bw=="
},
+ "@opentelemetry/api@1.9.0": {
+ "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="
+ },
"@types/estree@1.0.6": {
"integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw=="
},
@@ -113,6 +117,9 @@
"asynckit@0.4.0": {
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
},
+ "bintrees@1.0.2": {
+ "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw=="
+ },
"call-bind-apply-helpers@1.0.2": {
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
"dependencies": [
@@ -318,6 +325,13 @@
"entities"
]
},
+ "prom-client@15.1.3": {
+ "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==",
+ "dependencies": [
+ "@opentelemetry/api",
+ "tdigest"
+ ]
+ },
"punycode@2.3.1": {
"integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="
},
@@ -336,6 +350,12 @@
"symbol-tree@3.2.4": {
"integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="
},
+ "tdigest@0.1.2": {
+ "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==",
+ "dependencies": [
+ "bintrees"
+ ]
+ },
"tldts-core@6.1.84": {
"integrity": "sha512-NaQa1W76W2aCGjXybvnMYzGSM4x8fvG2AN/pla7qxcg0ZHbooOPhA8kctmOZUDfZyhDL27OGNbwAeig8P4p1vg=="
},
@@ -1102,7 +1122,8 @@
"jsr:@std/fs@1.0.14",
"jsr:@std/path@1.0.8",
"jsr:@std/toml@1.0.2",
- "npm:jsdom@26.0.0"
+ "npm:jsdom@26.0.0",
+ "npm:prom-client@15.1.3"
]
}
}
diff --git a/src/lib/helpers/config.ts b/src/lib/helpers/config.ts
index d539738..646132f 100644
--- a/src/lib/helpers/config.ts
+++ b/src/lib/helpers/config.ts
@@ -23,6 +23,13 @@ const ConfigSchema = z.object({
? false
: true,
),
+ enable_metrics: z.boolean().default(
+ Deno.env.get("ENABLE_METRICS") === "true"
+ ? true
+ : Deno.env.get("ENABLE_METRICS") === "false"
+ ? false
+ : true,
+ ),
}).strict().default({}),
cache: z.object({
enabled: z.boolean().default(true),
@@ -43,7 +50,9 @@ const ConfigSchema = z.object({
external_videoplayback_proxy: z.string().default(
Deno.env.get("EXTERNAL_VIDEOPLAYBACK_PROXY") || "",
),
- max_proxy_retries: z.number().default(Number(Deno.env.get("MAX_PROXY_RETIRES") || 5)),
+ max_proxy_retries: z.number().default(
+ Number(Deno.env.get("MAX_PROXY_RETIRES") || 5),
+ ),
}).strict().default({}),
jobs: z.object({
youtube_session: z.object({
diff --git a/src/lib/helpers/metrics.ts b/src/lib/helpers/metrics.ts
new file mode 100644
index 0000000..98ca83d
--- /dev/null
+++ b/src/lib/helpers/metrics.ts
@@ -0,0 +1,112 @@
+import { IRawResponse } from "youtubei.js";
+import { Counter, Registry } from "prom-client";
+
+export let metrics: Metrics | undefined;
+
+export class Metrics {
+ private METRICS_PREFIX = "invidious_companion_";
+ public register = new Registry();
+
+ public createCounter(name: string, help?: string): Counter {
+ return new Counter({
+ name: `${this.METRICS_PREFIX}${name}`,
+ help: help || "No help has been provided for this metric",
+ registers: [this.register],
+ });
+ }
+
+ public potokenGenerationFailure = this.createCounter(
+ "potoken_generation_failure",
+ "Number of times that the PoToken generation job has failed for whatever reason",
+ );
+
+ private innertubeErrorStatusUnknown = this.createCounter(
+ "innertube_error_status_unknown",
+ "Number of times that an unknown status has been returned by Innertube API",
+ );
+
+ private innertubeErrorReasonSignIn = this.createCounter(
+ "innertube_error_reason_SignIn",
+ 'Number of times that the message "Sign in to confirm you’re not a bot." has been returned by Innertube API',
+ );
+
+ private innertubeErrorSubreasonProtectCommunity = this.createCounter(
+ "innertube_error_subreason_ProtectCommunity",
+ 'Number of times that the message "This helps protect our community." has been returned by Innertube API',
+ );
+
+ private innertubeErrorReasonUnknown = this.createCounter(
+ "innertube_error_reason_unknown",
+ "Number of times that an unknown reason has been returned by the Innertube API",
+ );
+
+ private innertubeErrorSubreasonUnknown = this.createCounter(
+ "innertube_error_subreason_unknown",
+ "Number of times that an unknown subreason has been returned by the Innertube API",
+ );
+
+ public innertubeSuccessfulRequest = this.createCounter(
+ "innertube_successful_request",
+ "Number successful requests made to the Innertube API",
+ );
+
+ private innertubeFailedRequest = this.createCounter(
+ "innertube_failed_request",
+ "Number failed requests made to the Innertube API for whatever reason",
+ );
+
+ public checkInnertubeResponse(videoData: IRawResponse) {
+ this.innertubeFailedRequest.inc();
+
+ switch (true) {
+ // CONTENT_CHECK_REQUIRED: Sensitive content videos.
+ case (videoData.playabilityStatus?.status ===
+ "CONTENT_CHECK_REQUIRED"): {
+ break;
+ }
+ case (videoData.playabilityStatus?.status === "LOGIN_REQUIRED"): {
+ switch (true) {
+ // Age restricted videos, we don't need to track those.
+ case videoData.playabilityStatus?.reason?.includes(
+ "Sign in to confirm your age",
+ ): {
+ break;
+ }
+
+ case videoData.playabilityStatus?.reason?.includes(
+ "Sign in to confirm you’re not a bot",
+ ): {
+ this.innertubeErrorReasonSignIn.inc();
+
+ switch (true) {
+ case videoData.playabilityStatus?.errorScreen
+ ?.playerErrorMessageRenderer
+ ?.subreason?.runs?.[0]?.text?.includes(
+ "This helps protect our community",
+ ): {
+ this.innertubeErrorSubreasonProtectCommunity
+ .inc();
+ break;
+ }
+ default: {
+ this.innertubeErrorSubreasonUnknown.inc();
+ break;
+ }
+ }
+
+ break;
+ }
+
+ default: {
+ this.innertubeErrorReasonUnknown.inc();
+ break;
+ }
+ }
+ break;
+ }
+ default:
+ this.innertubeErrorStatusUnknown.inc();
+ break;
+ }
+ }
+}
diff --git a/src/lib/helpers/youtubePlayerHandling.ts b/src/lib/helpers/youtubePlayerHandling.ts
index 396eabf..c099026 100644
--- a/src/lib/helpers/youtubePlayerHandling.ts
+++ b/src/lib/helpers/youtubePlayerHandling.ts
@@ -2,6 +2,7 @@ import { ApiResponse, Innertube, YT } from "youtubei.js";
import { generateRandomString } from "youtubei.js/Utils";
import { compress, decompress } from "brotli";
import type { BG } from "bgutils";
+import { metrics } from "../helpers/metrics.ts";
let youtubePlayerReqLocation = "youtubePlayerReq";
if (Deno.env.get("YT_PLAYER_REQ_LOCATION")) {
if (Deno.env.has("DENO_COMPILED")) {
@@ -167,20 +168,25 @@ export const youtubePlayerParsing = async ({
microformat,
}))(videoData);
- if (cacheEnabled && videoData.playabilityStatus?.status == "OK") {
- (async () => {
- await kv.set(
- ["video_cache", videoId],
- compress(
- new TextEncoder().encode(
- JSON.stringify(videoOnlyNecessaryInfo),
+ if (videoData.playabilityStatus?.status == "OK") {
+ metrics?.innertubeSuccessfulRequest.inc();
+ if (cacheEnabled) {
+ (async () => {
+ await kv.set(
+ ["video_cache", videoId],
+ compress(
+ new TextEncoder().encode(
+ JSON.stringify(videoOnlyNecessaryInfo),
+ ),
),
- ),
- {
- expireIn: 1000 * 60 * 60,
- },
- );
- })();
+ {
+ expireIn: 1000 * 60 * 60,
+ },
+ );
+ })();
+ }
+ } else {
+ metrics?.checkInnertubeResponse(videoData);
}
return videoOnlyNecessaryInfo;
diff --git a/src/lib/types/HonoVariables.ts b/src/lib/types/HonoVariables.ts
index 4f42d28..1e9b7a2 100644
--- a/src/lib/types/HonoVariables.ts
+++ b/src/lib/types/HonoVariables.ts
@@ -1,9 +1,11 @@
import { Innertube } from "youtubei.js";
import { BG } from "bgutils";
import type { Config } from "../helpers/config.ts";
+import { Metrics } from "../helpers/metrics.ts";
export type HonoVariables = {
innertubeClient: Innertube;
config: Config;
tokenMinter: BG.WebPoMinter;
+ metrics: Metrics | undefined;
};
diff --git a/src/main.ts b/src/main.ts
index b57cdc5..51ec965 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -20,6 +20,7 @@ import type { HonoVariables } from "./lib/types/HonoVariables.ts";
import { parseConfig } from "./lib/helpers/config.ts";
const config = await parseConfig();
+import { Metrics } from "./lib/helpers/metrics.ts";
let getFetchClientLocation = "getFetchClient";
if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
@@ -38,6 +39,7 @@ declare module "hono" {
interface ContextVariableMap extends HonoVariables {}
}
const app = new Hono();
+const metrics = config.server.enable_metrics ? new Metrics() : undefined;
let tokenMinter: BG.WebPoMinter;
let innertubeClient: Innertube;
@@ -170,11 +172,16 @@ if (!innertubeClientOauthEnabled) {
{ backoffSchedule: [5_000, 15_000, 60_000, 180_000] },
async () => {
if (innertubeClientJobPoTokenEnabled) {
- ({ innertubeClient, tokenMinter } = await poTokenGenerate(
- innertubeClient,
- config,
- innertubeClientCache,
- ));
+ try {
+ ({ innertubeClient, tokenMinter } = await poTokenGenerate(
+ innertubeClient,
+ config,
+ innertubeClientCache,
+ ));
+ } catch (err) {
+ metrics?.potokenGenerationFailure.inc();
+ throw err;
+ }
} else {
innertubeClient = await Innertube.create({
enable_session_cache: false,
@@ -212,6 +219,7 @@ app.use("*", async (c, next) => {
c.set("innertubeClient", innertubeClient);
c.set("tokenMinter", tokenMinter);
c.set("config", config);
+ c.set("metrics", metrics);
await next();
});
diff --git a/src/routes/index.ts b/src/routes/index.ts
index 6448e3d..07ff900 100644
--- a/src/routes/index.ts
+++ b/src/routes/index.ts
@@ -11,6 +11,7 @@ import videoPlaybackProxy from "./videoPlaybackProxy.ts";
import health from "./health.ts";
import type { Config } from "../lib/helpers/config.ts";
import info from "./info.ts";
+import metrics from "./metrics.ts";
export const routes = (
app: Hono,
@@ -34,4 +35,7 @@ export const routes = (
app.route("/videoplayback", videoPlaybackProxy);
app.route("/healthz", health);
app.route("/info", info);
+ if (config.server.enable_metrics) {
+ app.route("/metrics", metrics);
+ }
};
diff --git a/src/routes/metrics.ts b/src/routes/metrics.ts
new file mode 100644
index 0000000..8e0eea8
--- /dev/null
+++ b/src/routes/metrics.ts
@@ -0,0 +1,11 @@
+import { Hono } from "hono";
+
+const metrics = new Hono();
+
+metrics.get("/", async (c) => {
+ return new Response(await c.get("metrics")?.register.metrics(), {
+ headers: { "Content-Type": "text/plain" },
+ });
+});
+
+export default metrics;
diff --git a/src/routes/youtube_api_routes/player.ts b/src/routes/youtube_api_routes/player.ts
index 0b4ac0e..a4071cc 100644
--- a/src/routes/youtube_api_routes/player.ts
+++ b/src/routes/youtube_api_routes/player.ts
@@ -7,6 +7,7 @@ player.post("/player", async (c) => {
const jsonReq = await c.req.json();
const innertubeClient = c.get("innertubeClient");
const config = c.get("config");
+ const metrics = c.get("metrics");
if (jsonReq.videoId) {
return c.json(
await youtubePlayerParsing({
@@ -14,6 +15,7 @@ player.post("/player", async (c) => {
videoId: jsonReq.videoId,
config,
tokenMinter: c.get("tokenMinter"),
+ metrics,
}),
);
}
--
2.49.0
Reference in a new issue View git blame Copy permalink