Compare commits
1 commit
master
...
sensitive-
Author | SHA1 | Date | |
---|---|---|---|
501bd507e5 |
21 changed files with 2853 additions and 1575 deletions
3
.github/workflows/deno-check.yaml
vendored
3
.github/workflows/deno-check.yaml
vendored
|
@ -19,9 +19,6 @@ jobs:
|
|||
|
||||
- name: Verify formatting
|
||||
run: deno fmt --check src/**
|
||||
|
||||
- name: Verify typing
|
||||
run: deno check src/**
|
||||
|
||||
- name: Run linter
|
||||
run: deno lint
|
||||
|
|
15
Dockerfile
15
Dockerfile
|
@ -1,4 +1,4 @@
|
|||
FROM denoland/deno:debian-2.1.10 AS builder
|
||||
FROM denoland/deno:debian-2.1.8 AS builder
|
||||
|
||||
# Fuck deno
|
||||
ENV RUST_MIN_STACK=9999999999
|
||||
|
@ -19,31 +19,22 @@ RUN curl -fsSL https://github.com/krallin/tini/releases/download/v${TINI_VERSION
|
|||
--output /tini \
|
||||
&& chmod +x /tini
|
||||
|
||||
RUN arch=$(uname -m) && \
|
||||
curl -fsSL https://github.com/dmikusa/tiny-health-checker/releases/download/v0.36.0/thc-${arch}-unknown-linux-musl \
|
||||
--output /thc \
|
||||
&& chmod +x /thc
|
||||
|
||||
RUN deno task compile
|
||||
|
||||
# Stage for creating the non-privileged user
|
||||
FROM alpine:3.20 AS user-stage
|
||||
FROM i.sanxian.tech/alpine:3.20 AS user-stage
|
||||
|
||||
RUN adduser -u 10001 -S appuser
|
||||
|
||||
FROM gcr.io/distroless/cc
|
||||
|
||||
COPY --from=builder /app/invidious_companion /app/
|
||||
COPY --from=builder /thc /thc
|
||||
COPY ./config/ /app/config/
|
||||
COPY --from=builder /tini /tini
|
||||
|
||||
ENV PORT=8282 \
|
||||
HOST=0.0.0.0
|
||||
|
||||
ENV THC_PORT=${PORT} \
|
||||
THC_PATH=/healthz
|
||||
|
||||
# Copy passwd file for the non-privileged user from the user-stage
|
||||
COPY --from=user-stage /etc/passwd /etc/passwd
|
||||
|
||||
|
@ -57,5 +48,3 @@ WORKDIR /app
|
|||
USER appuser
|
||||
|
||||
ENTRYPOINT ["/tini", "--", "/app/invidious_companion"]
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=5s --start-period=10s --retries=5 CMD ["/thc"]
|
||||
|
|
|
@ -3,9 +3,9 @@ port = 8282
|
|||
host = "127.0.0.1"
|
||||
# secret key needs to be 16 characters long or more
|
||||
secret_key = "CHANGE_ME"
|
||||
base_url = "http://localhost:8282"
|
||||
verify_requests = false
|
||||
# max_dash_resolution = 1080
|
||||
# encrypt_query_params = false
|
||||
|
||||
[cache]
|
||||
enabled = true
|
||||
|
@ -19,11 +19,6 @@ proxy_file = "proxies.txt"
|
|||
# Enable YouTube new video format UMP
|
||||
ump = false
|
||||
# external_videoplayback_proxy = ""
|
||||
# fetch_timeout_ms = 10000
|
||||
# fetch_retry_enable = true
|
||||
# fetch_retry_times = 3
|
||||
# fetch_retry_initial_debounce = 500
|
||||
# fetch_retry_debounce_multiplier = 2
|
||||
|
||||
[jobs]
|
||||
|
||||
|
|
24
deno.json
24
deno.json
|
@ -1,18 +1,17 @@
|
|||
{
|
||||
"tasks": {
|
||||
"dev": "deno run --allow-import=github.com:443,jsr.io:443,raw.githubusercontent.com:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname --allow-read --allow-write=/var/tmp/youtubei.js --watch --unsafely-ignore-certificate-errors src/main.ts",
|
||||
"dev": "deno run --allow-import=github.com:443,jsr.io:443,raw.githubusercontent.com:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-sys=hostname --allow-read --allow-write=/var/tmp/youtubei.js --watch src/main.ts",
|
||||
"compile": "deno compile --include ./src/lib/helpers/youtubePlayerReq.ts --include ./src/lib/helpers/getFetchClient.ts --output invidious_companion --allow-import=github.com:443,jsr.io:443,raw.githubusercontent.com:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-read --allow-sys=hostname --allow-write=/var/tmp/youtubei.js --unsafely-ignore-certificate-errors src/main.ts"
|
||||
},
|
||||
"imports": {
|
||||
"hono": "jsr:@hono/hono@^4.7.2",
|
||||
"hono/logger": "jsr:@hono/hono@^4.7.2/logger",
|
||||
"hono/bearer-auth": "jsr:@hono/hono@^4.7.2/bearer-auth",
|
||||
"hono": "jsr:@hono/hono@^4.6.5",
|
||||
"hono/logger": "jsr:@hono/hono@^4.6.5/logger",
|
||||
"hono/bearer-auth": "jsr:@hono/hono@^4.6.5/bearer-auth",
|
||||
"prom-client": "npm:prom-client@^15.1.3",
|
||||
"youtubei.js": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno.ts",
|
||||
"youtubei.js/Utils": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/utils/Utils.ts",
|
||||
"youtubei.js/NavigationEndpoint": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/parser/classes/NavigationEndpoint.ts",
|
||||
"youtubei.js/PlayerCaptionsTracklist": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/parser/classes/PlayerCaptionsTracklist.ts",
|
||||
"jsdom": "npm:jsdom@26.0.0",
|
||||
"youtubei.js": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v12.2.0-deno/deno.ts",
|
||||
"youtubei.js/Utils": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v12.2.0-deno/deno/src/utils/Utils.ts",
|
||||
"youtubei.js/NavigationEndpoint": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v12.2.0-deno/deno/src/parser/classes/NavigationEndpoint.ts",
|
||||
"jsdom": "npm:jsdom@25.0.1",
|
||||
"bgutils": "https://esm.sh/bgutils-js@3.1.0",
|
||||
"estree": "https://esm.sh/@types/estree@1.0.6",
|
||||
"@willsoto/node-konfig-core": "npm:@willsoto/node-konfig-core@5.0.0",
|
||||
|
@ -20,15 +19,12 @@
|
|||
"@willsoto/node-konfig-toml-parser": "npm:@willsoto/node-konfig-toml-parser@3.0.0",
|
||||
"youtubePlayerReq": "./src/lib/helpers/youtubePlayerReq.ts",
|
||||
"getFetchClient": "./src/lib/helpers/getFetchClient.ts",
|
||||
"metrics": "./src/routes/metrics.ts",
|
||||
"googlevideo": "jsr:@luanrt/googlevideo@^2.0.0",
|
||||
"jsr:@luanrt/jintr": "jsr:@luanrt/jintr@^3.2.1"
|
||||
"googlevideo": "npm:googlevideo@2.0.0"
|
||||
},
|
||||
"unstable": [
|
||||
"cron",
|
||||
"kv",
|
||||
"http",
|
||||
"temporal"
|
||||
"http"
|
||||
],
|
||||
"fmt": {
|
||||
"indentWidth": 4
|
||||
|
|
|
@ -1,61 +0,0 @@
|
|||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import { decodeBase64, encodeBase64 } from "jsr:@std/encoding/base64";
|
||||
import { Aes } from "https://deno.land/x/crypto@v0.10.1/aes.ts";
|
||||
import {
|
||||
Ecb,
|
||||
Padding,
|
||||
} from "https://deno.land/x/crypto@v0.10.1/block-modes.ts";
|
||||
|
||||
export const encryptQuery = (
|
||||
queryParams: string,
|
||||
konfigStore: Store,
|
||||
): string => {
|
||||
try {
|
||||
const cipher = new Ecb(
|
||||
Aes,
|
||||
new TextEncoder().encode((
|
||||
Deno.env.get("SERVER_SECRET_KEY") ||
|
||||
konfigStore.get("server.secret_key") as string
|
||||
).substring(0, 16)),
|
||||
Padding.PKCS7,
|
||||
);
|
||||
|
||||
const encodedData = new TextEncoder().encode(
|
||||
queryParams
|
||||
);
|
||||
|
||||
const encryptedData = cipher.encrypt(encodedData)
|
||||
|
||||
return encodeBase64(encryptedData).replace(/\+/g, "-").replace(/\//g, "_")
|
||||
} catch (_) {
|
||||
return ""
|
||||
}
|
||||
};
|
||||
|
||||
export const decryptQuery = (
|
||||
queryParams: string,
|
||||
konfigStore: Store,
|
||||
): string => {
|
||||
try {
|
||||
const decipher = new Ecb(
|
||||
Aes,
|
||||
new TextEncoder().encode((
|
||||
Deno.env.get("SERVER_SECRET_KEY") ||
|
||||
konfigStore.get("server.secret_key") as string
|
||||
).substring(0, 16)),
|
||||
Padding.PKCS7,
|
||||
);
|
||||
|
||||
const decryptedData = new TextDecoder().decode(
|
||||
decipher.decrypt(
|
||||
decodeBase64(
|
||||
queryParams.replace(/-/g, "+").replace(/_/g, "/"),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
return decryptedData
|
||||
} catch (_) {
|
||||
return ""
|
||||
}
|
||||
};
|
|
@ -1,11 +1,4 @@
|
|||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import { retry, type RetryOptions } from "jsr:@std/async";
|
||||
|
||||
type FetchInputParameter = Parameters<typeof fetch>[0];
|
||||
type FetchInitParameterWithClient =
|
||||
| RequestInit
|
||||
| RequestInit & { client: Deno.HttpClient };
|
||||
type FetchReturn = ReturnType<typeof fetch>;
|
||||
|
||||
let proxies: string[] = [];
|
||||
let currentProxyIndex = 0;
|
||||
|
@ -15,24 +8,27 @@ try {
|
|||
proxies = proxyData.split("\n").map((line) => line.trim()).filter((line) =>
|
||||
line.length > 0
|
||||
);
|
||||
console.log("[INFO] Num of proxies on the list:", proxies.length)
|
||||
} catch (error) {
|
||||
console.error("[ERROR] Error reading proxy file:", error);
|
||||
console.log("[INFO] Proxies from a list of proxies will not be used");
|
||||
}
|
||||
|
||||
export const getFetchClient = (konfigStore: Store): {
|
||||
(input: RequestInfo | URL, init?: RequestInit): Promise<Response>;
|
||||
(
|
||||
input: FetchInputParameter,
|
||||
init?: FetchInitParameterWithClient,
|
||||
): FetchReturn;
|
||||
input: Request | URL | string,
|
||||
init?: RequestInit & {
|
||||
client: Deno.HttpClient;
|
||||
},
|
||||
): Promise<Response>;
|
||||
(input: URL | Request | string, init?: RequestInit): Promise<Response>;
|
||||
} => {
|
||||
if (
|
||||
Deno.env.get("PROXY") || konfigStore.get("networking.proxy") ||
|
||||
(proxies.length > 0)
|
||||
) {
|
||||
return async (
|
||||
input: FetchInputParameter,
|
||||
input: RequestInfo | URL,
|
||||
init?: RequestInit,
|
||||
) => {
|
||||
const proxyUrl = Deno.env.get("PROXY") ||
|
||||
|
@ -63,39 +59,5 @@ export const getFetchClient = (konfigStore: Store): {
|
|||
};
|
||||
}
|
||||
|
||||
return (input: FetchInputParameter, init?: FetchInitParameterWithClient) =>
|
||||
fetchShim(konfigStore, input, init);
|
||||
return globalThis.fetch;
|
||||
};
|
||||
|
||||
function fetchShim(
|
||||
konfigStore: Store,
|
||||
input: FetchInputParameter,
|
||||
init?: FetchInitParameterWithClient,
|
||||
): FetchReturn {
|
||||
const fetchTimeout = konfigStore.get("networking.fetch_timeout_ms");
|
||||
const fetchRetry = konfigStore.get("networking.fetch_retry_enable");
|
||||
const fetchMaxAttempts = konfigStore.get("networking.fetch_retry_times");
|
||||
const fetchInitialDebounce = konfigStore.get(
|
||||
"networking.fetch_retry_initial_debounce",
|
||||
);
|
||||
const fetchDebounceMultiplier = konfigStore.get(
|
||||
"networking.fetch_retry_debounce_multiplier",
|
||||
);
|
||||
const retryOptions: RetryOptions = {
|
||||
maxAttempts: Number(fetchMaxAttempts) || 1,
|
||||
minTimeout: Number(fetchInitialDebounce) || 0,
|
||||
multiplier: Number(fetchDebounceMultiplier) || 0,
|
||||
jitter: 0,
|
||||
};
|
||||
|
||||
const callFetch = () =>
|
||||
fetch(input, {
|
||||
// only set the AbortSignal if the timeout is supplied in the config
|
||||
signal: fetchTimeout
|
||||
? AbortSignal.timeout(Number(fetchTimeout))
|
||||
: null,
|
||||
...(init || {}),
|
||||
});
|
||||
// if retry enabled, call retry with the fetch shim, otherwise pass the fetch shim back directly
|
||||
return fetchRetry ? retry(callFetch, retryOptions) : callFetch();
|
||||
}
|
||||
|
|
|
@ -2,9 +2,7 @@ import { ApiResponse, Innertube, YT } from "youtubei.js";
|
|||
import { generateRandomString } from "youtubei.js/Utils";
|
||||
import { compress, decompress } from "https://deno.land/x/brotli@0.1.7/mod.ts";
|
||||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import { failedRequests, retryCount, successfulRequests } from "metrics";
|
||||
import { innertubeEmbeddedClient } from "../../main.ts";
|
||||
import type { BG } from "bgutils";
|
||||
import { failedRequests, successfulRequests } from "../../routes/index.ts";
|
||||
let youtubePlayerReqLocation = "youtubePlayerReq";
|
||||
if (Deno.env.get("YT_PLAYER_REQ_LOCATION")) {
|
||||
if (Deno.env.has("DENO_COMPILED")) {
|
||||
|
@ -21,8 +19,6 @@ const { youtubePlayerReq } = await import(youtubePlayerReqLocation);
|
|||
const videoCachePath = Deno.env.get("VIDEO_CACHE_PATH") as string ||
|
||||
"/var/tmp/youtubei.js/video_cache";
|
||||
|
||||
const maxRetries = Deno.env.get("MAX_PROXY_RETRIES") as string || 3;
|
||||
|
||||
export let kv: Deno.Kv;
|
||||
if ((Deno.env.get("VIDEO_CACHE_ON_DISK")?.toLowerCase() ?? false) == "true") {
|
||||
console.log("[INFO] Storing video cache on disk");
|
||||
|
@ -31,84 +27,30 @@ if ((Deno.env.get("VIDEO_CACHE_ON_DISK")?.toLowerCase() ?? false) == "true") {
|
|||
kv = await Deno.openKv();
|
||||
}
|
||||
|
||||
export const youtubePlayerParsing = async ({
|
||||
innertubeClient,
|
||||
videoId,
|
||||
konfigStore,
|
||||
tokenMinter,
|
||||
overrideCache = false,
|
||||
}: {
|
||||
innertubeClient: Innertube;
|
||||
videoId: string;
|
||||
konfigStore: Store;
|
||||
tokenMinter: BG.WebPoMinter;
|
||||
overrideCache?: boolean;
|
||||
}): Promise<object> => {
|
||||
const cacheEnabled = overrideCache
|
||||
? false
|
||||
: konfigStore.get("cache.enabled");
|
||||
export const youtubePlayerParsing = async (
|
||||
innertubeClient: Innertube,
|
||||
videoId: string,
|
||||
konfigStore: Store,
|
||||
): Promise<object> => {
|
||||
const cacheEnabled = konfigStore.get("cache.enabled");
|
||||
|
||||
const videoCached = (await kv.get(["video_cache", videoId]))
|
||||
.value as Uint8Array;
|
||||
|
||||
if (videoCached != null && cacheEnabled) {
|
||||
if (videoCached != null && cacheEnabled == true) {
|
||||
return JSON.parse(new TextDecoder().decode(decompress(videoCached)));
|
||||
} else {
|
||||
let youtubePlayerResponse = await youtubePlayerReq(
|
||||
const youtubePlayerResponse = await youtubePlayerReq(
|
||||
innertubeClient,
|
||||
videoId,
|
||||
konfigStore,
|
||||
tokenMinter,
|
||||
);
|
||||
|
||||
for (let retries = 1; retries <= (maxRetries as number); retries++) {
|
||||
if (
|
||||
!youtubePlayerResponse.data.playabilityStatus?.errorScreen
|
||||
?.playerErrorMessageRenderer?.subreason?.runs?.[0]?.text
|
||||
?.includes("This helps protect our community")
|
||||
) {
|
||||
break;
|
||||
}
|
||||
console.log(
|
||||
`[DEBUG] Got 'This helps protect our community', retrying request for ${videoId}. Retry ${retries} of ${maxRetries}`,
|
||||
);
|
||||
retryCount.inc();
|
||||
youtubePlayerResponse = await youtubePlayerReq(
|
||||
innertubeClient,
|
||||
videoId,
|
||||
konfigStore,
|
||||
tokenMinter,
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
youtubePlayerResponse.data.playabilityStatus.status ===
|
||||
"UNPLAYABLE" ||
|
||||
youtubePlayerResponse.data.playabilityStatus.status ===
|
||||
"LOGIN_REQUIRED" ||
|
||||
youtubePlayerResponse.data.playabilityStatus.status ===
|
||||
"CONTENT_CHECK_REQUIRED" ||
|
||||
youtubePlayerResponse.data.playabilityStatus.reason ===
|
||||
"Sign in to confirm your age"
|
||||
) {
|
||||
innertubeEmbeddedClient.session.context.client.visitorData =
|
||||
innertubeClient.session.context.client.visitorData;
|
||||
if (innertubeClient.session.po_token) {
|
||||
innertubeEmbeddedClient.session.po_token =
|
||||
innertubeClient.session.po_token;
|
||||
}
|
||||
innertubeEmbeddedClient.session.player =
|
||||
innertubeClient.session.player;
|
||||
|
||||
youtubePlayerResponse = await youtubePlayerReq(
|
||||
innertubeEmbeddedClient,
|
||||
videoId,
|
||||
konfigStore,
|
||||
tokenMinter,
|
||||
);
|
||||
}
|
||||
|
||||
const videoData = youtubePlayerResponse.data;
|
||||
// console.log(videoData)
|
||||
|
||||
if (videoData.playabilityStatus.status == "ERROR") {
|
||||
throw("test")
|
||||
}
|
||||
|
||||
const video = new YT.VideoInfo(
|
||||
[youtubePlayerResponse],
|
||||
|
@ -116,6 +58,7 @@ export const youtubePlayerParsing = async ({
|
|||
generateRandomString(16),
|
||||
);
|
||||
|
||||
|
||||
const streamingData = video.streaming_data;
|
||||
|
||||
// Modify the original YouTube response to include deciphered URLs
|
||||
|
@ -206,6 +149,15 @@ export const youtubePlayerParsing = async ({
|
|||
streamingData,
|
||||
videoDetails,
|
||||
microformat,
|
||||
invidiousCompanion: {
|
||||
"baseUrl": Deno.env.get("SERVER_BASE_URL") ||
|
||||
konfigStore.get("server.base_url") as string,
|
||||
"external_videoplayback_proxy":
|
||||
Deno.env.get("EXTERNAL_VIDEOPLAYBACK_PROXY") ||
|
||||
konfigStore.get(
|
||||
"networking.external_videoplayback_proxy",
|
||||
) as string,
|
||||
},
|
||||
}))(videoData);
|
||||
|
||||
if (videoData.playabilityStatus?.status == "OK") {
|
||||
|
@ -226,9 +178,9 @@ export const youtubePlayerParsing = async ({
|
|||
},
|
||||
);
|
||||
})();
|
||||
} else {
|
||||
failedRequests.inc();
|
||||
}
|
||||
} else {
|
||||
failedRequests.inc();
|
||||
}
|
||||
|
||||
return videoOnlyNecessaryInfo;
|
||||
|
|
|
@ -1,13 +1,11 @@
|
|||
import { ApiResponse, ClientType, Innertube } from "youtubei.js";
|
||||
import { ApiResponse, Innertube } from "youtubei.js";
|
||||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import NavigationEndpoint from "youtubei.js/NavigationEndpoint";
|
||||
import type { BG } from "bgutils";
|
||||
|
||||
export const youtubePlayerReq = async (
|
||||
innertubeClient: Innertube,
|
||||
videoId: string,
|
||||
konfigStore: Store,
|
||||
tokenMinter: BG.WebPoMinter,
|
||||
): Promise<ApiResponse> => {
|
||||
const innertubeClientOauthEnabled = konfigStore.get(
|
||||
"youtube_session.oauth_enabled",
|
||||
|
@ -17,17 +15,12 @@ export const youtubePlayerReq = async (
|
|||
if (innertubeClientOauthEnabled) {
|
||||
innertubeClientUsed = "TV";
|
||||
}
|
||||
if (innertubeClient.session.client_name == ClientType.WEB_EMBEDDED) {
|
||||
innertubeClientUsed = "WEB_EMBEDDED"
|
||||
}
|
||||
|
||||
const watch_endpoint = new NavigationEndpoint({
|
||||
watchEndpoint: { videoId: videoId },
|
||||
});
|
||||
|
||||
const contentPoToken = await tokenMinter.mintAsWebsafeString(videoId);
|
||||
|
||||
return watch_endpoint.call(innertubeClient.actions, {
|
||||
return await watch_endpoint.call(innertubeClient.actions, {
|
||||
playbackContext: {
|
||||
contentPlaybackContext: {
|
||||
vis: 0,
|
||||
|
@ -37,7 +30,7 @@ export const youtubePlayerReq = async (
|
|||
},
|
||||
},
|
||||
serviceIntegrityDimensions: {
|
||||
poToken: contentPoToken,
|
||||
poToken: innertubeClient.session.po_token,
|
||||
},
|
||||
client: innertubeClientUsed,
|
||||
});
|
||||
|
|
|
@ -1,88 +0,0 @@
|
|||
import { Innertube } from "youtubei.js";
|
||||
import type { CaptionTrackData } from "youtubei.js/PlayerCaptionsTracklist";
|
||||
import { HTTPException } from "hono/http-exception";
|
||||
|
||||
function createTemporalDuration(milliseconds: number) {
|
||||
return new Temporal.Duration(
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
milliseconds,
|
||||
);
|
||||
}
|
||||
|
||||
const ESCAPE_SUBSTITUTIONS = {
|
||||
"&": "&",
|
||||
"<": "<",
|
||||
">": ">",
|
||||
"\u200E": "‎",
|
||||
"\u200F": "‏",
|
||||
"\u00A0": " ",
|
||||
};
|
||||
|
||||
export async function handleTranscripts(
|
||||
innertubeClient: Innertube,
|
||||
videoId: string,
|
||||
selectedCaption: CaptionTrackData,
|
||||
) {
|
||||
const lines: string[] = ["WEBVTT"];
|
||||
|
||||
const info = await innertubeClient.getInfo(videoId);
|
||||
const transcriptInfo = await (await info.getTranscript()).selectLanguage(
|
||||
selectedCaption.name.text || "",
|
||||
);
|
||||
const rawTranscriptLines = transcriptInfo.transcript.content?.body
|
||||
?.initial_segments;
|
||||
|
||||
if (rawTranscriptLines == undefined) throw new HTTPException(404);
|
||||
|
||||
rawTranscriptLines.forEach((line) => {
|
||||
const timestampFormatOptions = {
|
||||
style: "digital",
|
||||
minutesDisplay: "always",
|
||||
fractionalDigits: 3,
|
||||
};
|
||||
|
||||
// Temporal.Duration.prototype.toLocaleString() is supposed to delegate to Intl.DurationFormat
|
||||
// which Deno does not support. However, instead of following specs and having toLocaleString return
|
||||
// the same toString() it seems to have its own implementation of Intl.DurationFormat,
|
||||
// with its options parameter type incorrectly restricted to the same as the one for Intl.DateTimeFormatOptions
|
||||
// even though they do not share the same arguments.
|
||||
//
|
||||
// The above matches the options parameter of Intl.DurationFormat, and the resulting output is as expected.
|
||||
// Until this is fixed typechecking must be disabled for the two use cases below
|
||||
//
|
||||
// See
|
||||
// https://docs.deno.com/api/web/~/Intl.DateTimeFormatOptions
|
||||
// https://docs.deno.com/api/web/~/Temporal.Duration.prototype.toLocaleString
|
||||
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Temporal/Duration/toLocaleString
|
||||
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/DurationFormat/DurationFormat
|
||||
|
||||
const start_ms = createTemporalDuration(Number(line.start_ms)).round({
|
||||
largestUnit: "year",
|
||||
//@ts-ignore see above
|
||||
}).toLocaleString("en-US", timestampFormatOptions);
|
||||
|
||||
const end_ms = createTemporalDuration(Number(line.end_ms)).round({
|
||||
largestUnit: "year",
|
||||
//@ts-ignore see above
|
||||
}).toLocaleString("en-US", timestampFormatOptions);
|
||||
const timestamp = `${start_ms} --> ${end_ms}`;
|
||||
|
||||
const text = (line.snippet?.text || "").replace(
|
||||
/[&<>\u200E\u200F\u00A0]/g,
|
||||
(match: string) =>
|
||||
ESCAPE_SUBSTITUTIONS[
|
||||
match as keyof typeof ESCAPE_SUBSTITUTIONS
|
||||
],
|
||||
);
|
||||
|
||||
lines.push(`${timestamp}\n${text}`);
|
||||
});
|
||||
|
||||
return lines.join("\n\n");
|
||||
}
|
|
@ -1,13 +1,9 @@
|
|||
import { BG, buildURL, GOOG_API_KEY, USER_AGENT } from "bgutils";
|
||||
import type { WebPoSignalOutput } from "bgutils";
|
||||
import { BG } from "bgutils";
|
||||
import type { BgConfig } from "bgutils";
|
||||
import { JSDOM } from "jsdom";
|
||||
import { Innertube, UniversalCache } from "youtubei.js";
|
||||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import { poTokenFail } from "metrics";
|
||||
import {
|
||||
youtubePlayerParsing,
|
||||
youtubeVideoInfo,
|
||||
} from "../helpers/youtubePlayerHandling.ts";
|
||||
import { externalTokenGeneratorFail, poTokenFail } from "../../routes/index.ts";
|
||||
let getFetchClientLocation = "getFetchClient";
|
||||
if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
|
||||
if (Deno.env.has("DENO_COMPILED")) {
|
||||
|
@ -26,16 +22,66 @@ export const poTokenGenerate = async (
|
|||
innertubeClient: Innertube,
|
||||
konfigStore: Store<Record<string, unknown>>,
|
||||
innertubeClientCache: UniversalCache,
|
||||
): Promise<{ innertubeClient: Innertube; tokenMinter: BG.WebPoMinter }> => {
|
||||
if (innertubeClient.session.po_token) {
|
||||
innertubeClient = await Innertube.create({
|
||||
enable_session_cache: false,
|
||||
user_agent: USER_AGENT,
|
||||
retrieve_player: false,
|
||||
});
|
||||
): Promise<Innertube> => {
|
||||
const externalTokenGenerator = konfigStore.get(
|
||||
"server.external_token_generator",
|
||||
) as string;
|
||||
const externalTokenGeneratorKey = konfigStore.get(
|
||||
"server.external_token_generator_key",
|
||||
) as string;
|
||||
const requestKey = "O43z0dpjhgX20SCx4KAo";
|
||||
|
||||
if (externalTokenGenerator != "" && externalTokenGenerator != undefined) {
|
||||
console.log(
|
||||
"poTokenGenerate: using external token generator at " +
|
||||
externalTokenGenerator,
|
||||
);
|
||||
try {
|
||||
let response: Response;
|
||||
if (
|
||||
externalTokenGeneratorKey != "" &&
|
||||
externalTokenGeneratorKey != undefined
|
||||
) {
|
||||
response = await fetch(
|
||||
`${externalTokenGenerator}/generate`,
|
||||
{
|
||||
headers: {
|
||||
"Authorization":
|
||||
`Bearer ${externalTokenGeneratorKey}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
if (response.status == 401) {
|
||||
throw new Error(
|
||||
`Key '${externalTokenGeneratorKey}' is invalid!`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
response = await fetch(
|
||||
`${externalTokenGenerator}/generate`,
|
||||
);
|
||||
}
|
||||
const data = await response.json();
|
||||
return (await Innertube.create({
|
||||
po_token: data.potoken,
|
||||
visitor_data: data.visitorData,
|
||||
fetch: getFetchClient(konfigStore),
|
||||
cache: innertubeClientCache,
|
||||
generate_session_locally: true,
|
||||
}));
|
||||
} catch (e) {
|
||||
console.error(
|
||||
"poTokenGenerate: error fetch token from the external token generator: " +
|
||||
e,
|
||||
);
|
||||
console.log("poTokenGenerate: Using built-in token generator");
|
||||
externalTokenGeneratorFail.inc();
|
||||
}
|
||||
}
|
||||
|
||||
const fetchImpl = await getFetchClient(konfigStore);
|
||||
if (innertubeClient.session.po_token) {
|
||||
innertubeClient = await Innertube.create({ retrieve_player: false });
|
||||
}
|
||||
|
||||
const visitorData = innertubeClient.session.context.client.visitorData;
|
||||
|
||||
|
@ -43,41 +89,29 @@ export const poTokenGenerate = async (
|
|||
throw new Error("Could not get visitor data");
|
||||
}
|
||||
|
||||
const dom = new JSDOM(
|
||||
'<!DOCTYPE html><html lang="en"><head><title></title></head><body></body></html>',
|
||||
{
|
||||
url: "https://www.youtube.com/",
|
||||
referrer: "https://www.youtube.com/",
|
||||
userAgent: USER_AGENT,
|
||||
},
|
||||
);
|
||||
const dom = new JSDOM();
|
||||
|
||||
Object.assign(globalThis, {
|
||||
window: dom.window,
|
||||
document: dom.window.document,
|
||||
location: dom.window.location,
|
||||
origin: dom.window.origin,
|
||||
});
|
||||
|
||||
if (!Reflect.has(globalThis, "navigator")) {
|
||||
Object.defineProperty(globalThis, "navigator", {
|
||||
value: dom.window.navigator,
|
||||
});
|
||||
}
|
||||
const bgConfig: BgConfig = {
|
||||
fetch: getFetchClient(konfigStore),
|
||||
globalObj: globalThis,
|
||||
identifier: visitorData,
|
||||
requestKey,
|
||||
};
|
||||
|
||||
const challengeResponse = await innertubeClient.getAttestationChallenge(
|
||||
"ENGAGEMENT_TYPE_UNBOUND",
|
||||
);
|
||||
if (!challengeResponse.bg_challenge) {
|
||||
const bgChallenge = await BG.Challenge.create(bgConfig);
|
||||
|
||||
if (!bgChallenge) {
|
||||
poTokenFail.inc();
|
||||
throw new Error("Could not get challenge");
|
||||
}
|
||||
|
||||
const interpreterUrl = challengeResponse.bg_challenge.interpreter_url
|
||||
.private_do_not_access_or_else_trusted_resource_url_wrapped_value;
|
||||
const bgScriptResponse = await fetchImpl(
|
||||
`http:${interpreterUrl}`,
|
||||
);
|
||||
const interpreterJavascript = await bgScriptResponse.text();
|
||||
const interpreterJavascript = bgChallenge.interpreterJavascript
|
||||
.privateDoNotAccessOrElseSafeScriptWrappedValue;
|
||||
|
||||
if (interpreterJavascript) {
|
||||
new Function(interpreterJavascript)();
|
||||
|
@ -86,79 +120,19 @@ export const poTokenGenerate = async (
|
|||
throw new Error("Could not load VM");
|
||||
}
|
||||
|
||||
// Botguard currently surfaces a "Not implemented" error here, due to the environment
|
||||
// not having a valid Canvas API in JSDOM. At the time of writing, this doesn't cause
|
||||
// any issues as the Canvas check doesn't appear to be an enforced element of the checks
|
||||
console.log(
|
||||
'[INFO] the "Not implemented: HTMLCanvasElement.prototype.getContext" error is normal. Please do not open a bug report about it.',
|
||||
);
|
||||
const botguard = await BG.BotGuardClient.create({
|
||||
program: challengeResponse.bg_challenge.program,
|
||||
globalName: challengeResponse.bg_challenge.global_name,
|
||||
globalObj: globalThis,
|
||||
const poTokenResult = await BG.PoToken.generate({
|
||||
program: bgChallenge.program,
|
||||
globalName: bgChallenge.globalName,
|
||||
bgConfig,
|
||||
});
|
||||
|
||||
const webPoSignalOutput: WebPoSignalOutput = [];
|
||||
const botguardResponse = await botguard.snapshot({ webPoSignalOutput });
|
||||
const requestKey = "O43z0dpjhgX20SCx4KAo";
|
||||
await BG.PoToken.generatePlaceholder(visitorData);
|
||||
|
||||
const integrityTokenResponse = await fetchImpl(
|
||||
buildURL("GenerateIT", true),
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json+protobuf",
|
||||
"x-goog-api-key": GOOG_API_KEY,
|
||||
"x-user-agent": "grpc-web-javascript/0.1",
|
||||
"user-agent": USER_AGENT,
|
||||
},
|
||||
body: JSON.stringify([requestKey, botguardResponse]),
|
||||
},
|
||||
);
|
||||
|
||||
const response = await integrityTokenResponse.json() as unknown[];
|
||||
|
||||
if (typeof response[0] !== "string") {
|
||||
throw new Error("Could not get integrity token");
|
||||
}
|
||||
|
||||
const integrityTokenBasedMinter = await BG.WebPoMinter.create({
|
||||
integrityToken: response[0],
|
||||
}, webPoSignalOutput);
|
||||
|
||||
const sessionPoToken = await integrityTokenBasedMinter.mintAsWebsafeString(
|
||||
visitorData,
|
||||
);
|
||||
|
||||
const instantiatedInnertubeClient = await Innertube.create({
|
||||
enable_session_cache: false,
|
||||
po_token: sessionPoToken,
|
||||
return (await Innertube.create({
|
||||
po_token: poTokenResult.poToken,
|
||||
visitor_data: visitorData,
|
||||
fetch: getFetchClient(konfigStore),
|
||||
cache: innertubeClientCache,
|
||||
generate_session_locally: true,
|
||||
});
|
||||
|
||||
try {
|
||||
const feed = await instantiatedInnertubeClient.getTrending();
|
||||
// get all videos and shuffle them randomly to avoid using the same trending video over and over
|
||||
const videos = feed.videos
|
||||
.filter((video) => video.type === "Video")
|
||||
.map((value) => ({ value, sort: Math.random() }))
|
||||
.sort((a, b) => a.sort - b.sort)
|
||||
.map(({ value }) => value);
|
||||
|
||||
const video = videos.find((video) => "id" in video);
|
||||
if (!video) {
|
||||
throw new Error("no videos with id found in trending");
|
||||
}
|
||||
} catch (err) {
|
||||
console.log("Failed to get valid PO token, will retry", { err });
|
||||
throw err;
|
||||
}
|
||||
|
||||
return {
|
||||
innertubeClient: instantiatedInnertubeClient,
|
||||
tokenMinter: integrityTokenBasedMinter,
|
||||
};
|
||||
}));
|
||||
};
|
||||
|
|
|
@ -1,9 +1,5 @@
|
|||
import { Innertube } from "youtubei.js";
|
||||
import { BG } from "bgutils";
|
||||
import type { konfigLoader } from "../helpers/konfigLoader.ts";
|
||||
|
||||
export type HonoVariables = {
|
||||
innertubeClient: Innertube;
|
||||
konfigStore: Awaited<ReturnType<typeof konfigLoader>>;
|
||||
tokenMinter: BG.WebPoMinter;
|
||||
};
|
||||
|
|
70
src/main.ts
70
src/main.ts
|
@ -1,13 +1,8 @@
|
|||
import { Hono } from "hono";
|
||||
import { routes } from "./routes/index.ts";
|
||||
import { ClientType, Innertube, UniversalCache } from "youtubei.js";
|
||||
import { Innertube, UniversalCache } from "youtubei.js";
|
||||
import { poTokenGenerate } from "./lib/jobs/potoken.ts";
|
||||
import { USER_AGENT } from "bgutils";
|
||||
import { konfigLoader } from "./lib/helpers/konfigLoader.ts";
|
||||
import { retry } from "jsr:@std/async";
|
||||
import type { HonoVariables } from "./lib/types/HonoVariables.ts";
|
||||
import type { BG } from "bgutils";
|
||||
|
||||
let getFetchClientLocation = "getFetchClient";
|
||||
if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
|
||||
if (Deno.env.has("DENO_COMPILED")) {
|
||||
|
@ -21,13 +16,9 @@ if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
|
|||
}
|
||||
const { getFetchClient } = await import(getFetchClientLocation);
|
||||
|
||||
declare module "hono" {
|
||||
interface ContextVariableMap extends HonoVariables {}
|
||||
}
|
||||
const app = new Hono();
|
||||
const konfigStore = await konfigLoader();
|
||||
|
||||
let tokenMinter: BG.WebPoMinter;
|
||||
let innertubeClient: Innertube;
|
||||
let innertubeClientFetchPlayer = true;
|
||||
const innertubeClientOauthEnabled = konfigStore.get(
|
||||
|
@ -60,21 +51,10 @@ if (!innertubeClientOauthEnabled) {
|
|||
}
|
||||
|
||||
innertubeClient = await Innertube.create({
|
||||
enable_session_cache: false,
|
||||
cache: innertubeClientCache,
|
||||
retrieve_player: innertubeClientFetchPlayer,
|
||||
fetch: getFetchClient(konfigStore),
|
||||
cookie: innertubeClientCookies || undefined,
|
||||
user_agent: USER_AGENT,
|
||||
});
|
||||
|
||||
export const innertubeEmbeddedClient = await Innertube.create({
|
||||
cache: innertubeClientCache,
|
||||
client_type: ClientType.WEB_EMBEDDED,
|
||||
retrieve_player: false,
|
||||
fetch: getFetchClient(konfigStore),
|
||||
cookie: innertubeClientCookies || undefined,
|
||||
user_agent: USER_AGENT,
|
||||
});
|
||||
|
||||
const poTokenRefreshInterval = Deno.env.get("FREQUENCY_SECONDS") ||
|
||||
|
@ -83,31 +63,24 @@ console.log("[INFO] po_token refresh interval set to", poTokenRefreshInterval);
|
|||
|
||||
if (!innertubeClientOauthEnabled) {
|
||||
if (innertubeClientJobPoTokenEnabled) {
|
||||
({ innertubeClient, tokenMinter } = await retry(
|
||||
poTokenGenerate.bind(
|
||||
poTokenGenerate,
|
||||
innertubeClient,
|
||||
konfigStore,
|
||||
innertubeClientCache as UniversalCache,
|
||||
),
|
||||
{ minTimeout: 1_000, maxTimeout: 60_000, multiplier: 5, jitter: 0 },
|
||||
));
|
||||
innertubeClient = await poTokenGenerate(
|
||||
innertubeClient,
|
||||
konfigStore,
|
||||
innertubeClientCache as UniversalCache,
|
||||
);
|
||||
}
|
||||
setInterval(
|
||||
async () => {
|
||||
if (innertubeClientJobPoTokenEnabled) {
|
||||
({ innertubeClient, tokenMinter } = await poTokenGenerate(
|
||||
innertubeClient = await poTokenGenerate(
|
||||
innertubeClient,
|
||||
konfigStore,
|
||||
innertubeClientCache,
|
||||
));
|
||||
);
|
||||
} else {
|
||||
innertubeClient = await Innertube.create({
|
||||
enable_session_cache: false,
|
||||
cache: innertubeClientCache,
|
||||
fetch: getFetchClient(konfigStore),
|
||||
retrieve_player: innertubeClientFetchPlayer,
|
||||
user_agent: USER_AGENT,
|
||||
});
|
||||
}
|
||||
},
|
||||
|
@ -136,16 +109,31 @@ if (!innertubeClientOauthEnabled) {
|
|||
}
|
||||
|
||||
app.use("*", async (c, next) => {
|
||||
// @ts-ignore Do not understand how to fix this error.
|
||||
c.set("innertubeClient", innertubeClient);
|
||||
c.set("tokenMinter", tokenMinter);
|
||||
// @ts-ignore Do not understand how to fix this error.
|
||||
c.set("konfigStore", konfigStore);
|
||||
await next();
|
||||
});
|
||||
|
||||
routes(app, konfigStore);
|
||||
|
||||
Deno.serve({
|
||||
port: Number(Deno.env.get("PORT")) ||
|
||||
konfigStore.get("server.port") as number,
|
||||
hostname: Deno.env.get("HOST") || konfigStore.get("server.host") as string,
|
||||
}, app.fetch);
|
||||
const https = Deno.env.get("HTTPS");
|
||||
const port = konfigStore.get("server.port") as number;
|
||||
const host = konfigStore.get("server.host") as string;
|
||||
|
||||
if (https == "TRUE" || https == "true") {
|
||||
const cert = Deno.readTextFileSync("/data/cert.pem");
|
||||
const key = Deno.readTextFileSync("/data/key.key");
|
||||
Deno.serve({
|
||||
port: port,
|
||||
hostname: host,
|
||||
cert: cert,
|
||||
key: key,
|
||||
}, app.fetch);
|
||||
} else {
|
||||
Deno.serve({
|
||||
port: port,
|
||||
hostname: host,
|
||||
}, app.fetch);
|
||||
}
|
||||
|
|
|
@ -2,15 +2,72 @@ import { Hono } from "hono";
|
|||
import { logger } from "hono/logger";
|
||||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import { bearerAuth } from "hono/bearer-auth";
|
||||
import { Counter, Gauge, Registry } from "prom-client";
|
||||
|
||||
import youtubeApiPlayer from "./youtube_api_routes/player.ts";
|
||||
import invidiousRouteLatestVersion from "./invidious_routes/latestVersion.ts";
|
||||
import invidiousRouteDashManifest from "./invidious_routes/dashManifest.ts";
|
||||
import invidiousCaptionsApi from "./invidious_routes/captions.ts";
|
||||
import videoPlaybackProxy from "./videoPlaybackProxy.ts";
|
||||
import metrics from "metrics";
|
||||
import metrics from "./metrics.ts";
|
||||
|
||||
const METRICS_PREFIX = "invidious_companion_";
|
||||
export const register = new Registry();
|
||||
|
||||
export const externalTokenGeneratorFail = new Counter({
|
||||
name: `${METRICS_PREFIX}externaltokengenerator_fail`,
|
||||
help: "TODO",
|
||||
registers: [register],
|
||||
});
|
||||
|
||||
export const cachedEntries = new Gauge({
|
||||
name: `${METRICS_PREFIX}cached_entries`,
|
||||
help: "TODO",
|
||||
registers: [register],
|
||||
});
|
||||
|
||||
export const subreasonProtectCommunity = new Counter({
|
||||
name: `${METRICS_PREFIX}subreason_protect_community`,
|
||||
help: "TODO",
|
||||
registers: [register],
|
||||
});
|
||||
|
||||
export const poTokenFail = new Counter({
|
||||
name: `${METRICS_PREFIX}potoken_fail`,
|
||||
help: "TODO",
|
||||
registers: [register],
|
||||
});
|
||||
|
||||
export const reasonBot = new Counter({
|
||||
name: `${METRICS_PREFIX}reason_bot`,
|
||||
help: "TODO",
|
||||
registers: [register],
|
||||
});
|
||||
|
||||
export const videoUnavailable = new Counter({
|
||||
name: `${METRICS_PREFIX}video_unavailable`,
|
||||
help: "TODO",
|
||||
registers: [register],
|
||||
});
|
||||
|
||||
export const videoRestricted = new Counter({
|
||||
name: `${METRICS_PREFIX}video_restricted`,
|
||||
help: "TODO",
|
||||
registers: [register],
|
||||
});
|
||||
|
||||
export const failedRequests = new Counter({
|
||||
name: `${METRICS_PREFIX}failed_requests`,
|
||||
help: "TODO",
|
||||
registers: [register],
|
||||
});
|
||||
|
||||
export const successfulRequests = new Counter({
|
||||
name: `${METRICS_PREFIX}successful_requests`,
|
||||
help: "TODO",
|
||||
registers: [register],
|
||||
});
|
||||
|
||||
import health from "./health.ts";
|
||||
import info from "./info.ts";
|
||||
|
||||
export const routes = (
|
||||
app: Hono,
|
||||
|
@ -29,18 +86,7 @@ export const routes = (
|
|||
app.route("/youtubei/v1", youtubeApiPlayer);
|
||||
app.route("/latest_version", invidiousRouteLatestVersion);
|
||||
app.route("/api/manifest/dash/id", invidiousRouteDashManifest);
|
||||
if (
|
||||
!Deno.env.get("EXTERNAL_VIDEOPLAYBACK_PROXY") &&
|
||||
!(konfigStore.get(
|
||||
"networking.external_videoplayback_proxy",
|
||||
) as string ?? undefined)
|
||||
) {
|
||||
app.route("/videoplayback", videoPlaybackProxy);
|
||||
}
|
||||
app.route("/metrics", metrics);
|
||||
app.route("/api/v1/captions", invidiousCaptionsApi);
|
||||
app.route("/videoplayback", videoPlaybackProxy);
|
||||
app.route("/metrics", metrics);
|
||||
app.route("/healthz", health);
|
||||
|
||||
app.route("/info", info);
|
||||
};
|
||||
|
|
|
@ -1,21 +0,0 @@
|
|||
import { Hono } from "hono";
|
||||
|
||||
const info = new Hono();
|
||||
|
||||
info.get("/", (c) => {
|
||||
const konfigStore = c.get("konfigStore");
|
||||
|
||||
const json = {
|
||||
"external_videoplayback_proxy":
|
||||
Deno.env.get("EXTERNAL_VIDEOPLAYBACK_PROXY") ||
|
||||
konfigStore.get(
|
||||
"networking.external_videoplayback_proxy",
|
||||
) as string,
|
||||
};
|
||||
return new Response(JSON.stringify(json), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
});
|
||||
|
||||
export default info;
|
|
@ -1,95 +0,0 @@
|
|||
import { Hono } from "hono";
|
||||
import type { HonoVariables } from "../../lib/types/HonoVariables.ts";
|
||||
import { verifyRequest } from "../../lib/helpers/verifyRequest.ts";
|
||||
import {
|
||||
youtubePlayerParsing,
|
||||
youtubeVideoInfo,
|
||||
} from "../../lib/helpers/youtubePlayerHandling.ts";
|
||||
import type { CaptionTrackData } from "youtubei.js/PlayerCaptionsTracklist";
|
||||
import { handleTranscripts } from "../../lib/helpers/youtubeTranscriptsHandling.ts";
|
||||
import { HTTPException } from "hono/http-exception";
|
||||
|
||||
interface AvailableCaption {
|
||||
label: string;
|
||||
languageCode: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
const captionsHandler = new Hono<{ Variables: HonoVariables }>();
|
||||
captionsHandler.get("/:videoId", async (c) => {
|
||||
const { videoId } = c.req.param();
|
||||
const konfigStore = c.get("konfigStore");
|
||||
|
||||
const check = c.req.query("check");
|
||||
|
||||
if (konfigStore.get("server.verify_requests") && check == undefined) {
|
||||
throw new HTTPException(400, {
|
||||
res: new Response("No check ID."),
|
||||
});
|
||||
} else if (konfigStore.get("server.verify_requests") && check) {
|
||||
if (verifyRequest(check, videoId, konfigStore) === false) {
|
||||
throw new HTTPException(400, {
|
||||
res: new Response("ID incorrect."),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const innertubeClient = c.get("innertubeClient");
|
||||
|
||||
const youtubePlayerResponseJson = await youtubePlayerParsing({
|
||||
innertubeClient,
|
||||
videoId,
|
||||
konfigStore,
|
||||
tokenMinter: c.get("tokenMinter"),
|
||||
});
|
||||
|
||||
const videoInfo = youtubeVideoInfo(
|
||||
innertubeClient,
|
||||
youtubePlayerResponseJson,
|
||||
);
|
||||
|
||||
const captionsTrackArray = videoInfo.captions?.caption_tracks;
|
||||
if (captionsTrackArray == undefined) throw new HTTPException(404);
|
||||
|
||||
const label = c.req.query("label");
|
||||
const lang = c.req.query("lang");
|
||||
|
||||
// Show all available captions when a specific one is not selected
|
||||
if (label == undefined && lang == undefined) {
|
||||
const invidiousAvailableCaptionsArr: AvailableCaption[] = [];
|
||||
|
||||
for (const caption_track of captionsTrackArray) {
|
||||
invidiousAvailableCaptionsArr.push({
|
||||
label: caption_track.name.text || "",
|
||||
languageCode: caption_track.language_code,
|
||||
url: `/api/v1/captions/${videoId}?label=${
|
||||
encodeURIComponent(caption_track.name.text || "")
|
||||
}`,
|
||||
});
|
||||
}
|
||||
|
||||
return c.json({ captions: invidiousAvailableCaptionsArr });
|
||||
}
|
||||
|
||||
// Extract selected caption
|
||||
let filterSelected: CaptionTrackData[];
|
||||
|
||||
if (lang) {
|
||||
filterSelected = captionsTrackArray.filter((c: CaptionTrackData) =>
|
||||
c.language_code === lang
|
||||
);
|
||||
} else {
|
||||
filterSelected = captionsTrackArray.filter((c: CaptionTrackData) =>
|
||||
c.name.text === label
|
||||
);
|
||||
}
|
||||
|
||||
if (filterSelected.length == 0) throw new HTTPException(404);
|
||||
|
||||
c.header("Content-Type", "text/vtt; charset=UTF-8");
|
||||
return c.body(
|
||||
await handleTranscripts(innertubeClient, videoId, filterSelected[0]),
|
||||
);
|
||||
});
|
||||
|
||||
export default captionsHandler;
|
|
@ -1,22 +1,26 @@
|
|||
import { Hono } from "hono";
|
||||
import { FormatUtils } from "youtubei.js";
|
||||
import { FormatUtils, Innertube } from "youtubei.js";
|
||||
import { HonoVariables } from "../../lib/types/HonoVariables.ts";
|
||||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import {
|
||||
youtubePlayerParsing,
|
||||
youtubeVideoInfo,
|
||||
} from "../../lib/helpers/youtubePlayerHandling.ts";
|
||||
import { verifyRequest } from "../../lib/helpers/verifyRequest.ts";
|
||||
import { HTTPException } from "hono/http-exception";
|
||||
import { encryptQuery } from "../../lib/helpers/encrypter.ts";
|
||||
|
||||
const dashManifest = new Hono();
|
||||
const dashManifest = new Hono<{ Variables: HonoVariables }>();
|
||||
|
||||
dashManifest.get("/:videoId", async (c) => {
|
||||
const { videoId } = c.req.param();
|
||||
const { check, local } = c.req.query();
|
||||
c.header("access-control-allow-origin", "*");
|
||||
|
||||
const innertubeClient = c.get("innertubeClient");
|
||||
const konfigStore = c.get("konfigStore");
|
||||
const innertubeClient = await c.get("innertubeClient") as Innertube;
|
||||
// @ts-ignore Do not understand how to fix this error.
|
||||
const konfigStore = await c.get("konfigStore") as Store<
|
||||
Record<string, unknown>
|
||||
>;
|
||||
|
||||
const maxDashResolution = Deno.env.get("SERVER_MAX_DASH_RESOLUTION") ||
|
||||
konfigStore.get("server.max_dash_resolution") as number;
|
||||
|
@ -36,12 +40,11 @@ dashManifest.get("/:videoId", async (c) => {
|
|||
}
|
||||
}
|
||||
|
||||
const youtubePlayerResponseJson = await youtubePlayerParsing({
|
||||
const youtubePlayerResponseJson = await youtubePlayerParsing(
|
||||
innertubeClient,
|
||||
videoId,
|
||||
konfigStore,
|
||||
tokenMinter: c.get("tokenMinter"),
|
||||
});
|
||||
);
|
||||
const videoInfo = youtubeVideoInfo(
|
||||
innertubeClient,
|
||||
youtubePlayerResponseJson,
|
||||
|
@ -95,31 +98,16 @@ dashManifest.get("/:videoId", async (c) => {
|
|||
videoInfo.page[0].video_details?.is_post_live_dvr,
|
||||
(url: URL) => {
|
||||
let dashUrl = url;
|
||||
let queryParams = dashUrl.search.substring(1) + "&host=" +
|
||||
dashUrl.host;
|
||||
|
||||
if (local) {
|
||||
if (konfigStore.get("networking.ump") as boolean) {
|
||||
queryParams = queryParams + "&ump=1";
|
||||
}
|
||||
if (
|
||||
Deno.env.get("ENCRYPT_QUERY_PARAMS") ||
|
||||
konfigStore.get("server.encrypt_query_params") as boolean
|
||||
) {
|
||||
queryParams = "enc=yes&data=" + encryptQuery(
|
||||
queryParams,
|
||||
konfigStore,
|
||||
);
|
||||
}
|
||||
// Can't create URL type without host part
|
||||
dashUrl = (
|
||||
Deno.env.get("EXTERNAL_VIDEOPLAYBACK_PROXY") ||
|
||||
(konfigStore.get(
|
||||
"networking.external_videoplayback_proxy",
|
||||
) as string ?? "")
|
||||
) +
|
||||
(dashUrl.pathname + "?" +
|
||||
queryParams) as unknown as URL;
|
||||
dashUrl = (konfigStore.get(
|
||||
"networking.external_videoplayback_proxy",
|
||||
) as string ?? "") +
|
||||
(dashUrl.pathname + dashUrl.search + "&host=" +
|
||||
dashUrl.host) as unknown as URL;
|
||||
if (konfigStore.get("networking.ump") as boolean) {
|
||||
dashUrl = dashUrl + "&ump=1" as unknown as URL;
|
||||
}
|
||||
return dashUrl;
|
||||
} else {
|
||||
return dashUrl;
|
||||
|
@ -133,7 +121,7 @@ dashManifest.get("/:videoId", async (c) => {
|
|||
captions,
|
||||
undefined,
|
||||
);
|
||||
return c.body(dashFile);
|
||||
return c.text(dashFile.replaceAll("&", "&"));
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
@ -1,13 +1,15 @@
|
|||
import { Hono } from "hono";
|
||||
import { HTTPException } from "hono/http-exception";
|
||||
import { Innertube } from "youtubei.js";
|
||||
import { HonoVariables } from "../../lib/types/HonoVariables.ts";
|
||||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import {
|
||||
youtubePlayerParsing,
|
||||
youtubeVideoInfo,
|
||||
} from "../../lib/helpers/youtubePlayerHandling.ts";
|
||||
import { verifyRequest } from "../../lib/helpers/verifyRequest.ts";
|
||||
import { encryptQuery } from "../../lib/helpers/encrypter.ts";
|
||||
|
||||
const latestVersion = new Hono();
|
||||
const latestVersion = new Hono<{ Variables: HonoVariables }>();
|
||||
|
||||
latestVersion.get("/", async (c) => {
|
||||
const { check, itag, id, local } = c.req.query();
|
||||
|
@ -19,14 +21,19 @@ latestVersion.get("/", async (c) => {
|
|||
});
|
||||
}
|
||||
|
||||
const innertubeClient = c.get("innertubeClient");
|
||||
const konfigStore = c.get("konfigStore");
|
||||
const innertubeClient = await c.get("innertubeClient") as Innertube;
|
||||
// @ts-ignore Do not understand how to fix this error.
|
||||
const konfigStore = await c.get("konfigStore") as Store<
|
||||
Record<string, unknown>
|
||||
>;
|
||||
const verifyRequests = Deno.env.get("VERIFY_REQUESTS") ||
|
||||
konfigStore.get("server.verify_requests");
|
||||
|
||||
if (konfigStore.get("server.verify_requests") && check == undefined) {
|
||||
if (verifyRequests && check == undefined) {
|
||||
throw new HTTPException(400, {
|
||||
res: new Response("No check ID."),
|
||||
});
|
||||
} else if (konfigStore.get("server.verify_requests") && check) {
|
||||
} else if (verifyRequests && check) {
|
||||
if (verifyRequest(check, id, konfigStore) === false) {
|
||||
throw new HTTPException(400, {
|
||||
res: new Response("ID incorrect."),
|
||||
|
@ -34,18 +41,17 @@ latestVersion.get("/", async (c) => {
|
|||
}
|
||||
}
|
||||
|
||||
const youtubePlayerResponseJson = await youtubePlayerParsing({
|
||||
const youtubePlayerResponseJson = await youtubePlayerParsing(
|
||||
innertubeClient,
|
||||
videoId: id,
|
||||
id,
|
||||
konfigStore,
|
||||
tokenMinter: c.get("tokenMinter"),
|
||||
});
|
||||
);
|
||||
const videoInfo = youtubeVideoInfo(
|
||||
innertubeClient,
|
||||
youtubePlayerResponseJson,
|
||||
);
|
||||
|
||||
if (videoInfo.playability_status?.status !== "OK") {
|
||||
if ((videoInfo.playability_status?.status !== "OK") && (videoInfo.playability_status?.status !== "CONTENT_CHECK_REQUIRED")) {
|
||||
throw ("The video can't be played: " + id + " due to reason: " +
|
||||
videoInfo.playability_status?.reason);
|
||||
}
|
||||
|
@ -64,24 +70,12 @@ latestVersion.get("/", async (c) => {
|
|||
const itagUrl = selectedItagFormat[0].url as string;
|
||||
const itagUrlParsed = new URL(itagUrl);
|
||||
let urlToRedirect = itagUrlParsed.toString();
|
||||
let queryParams = itagUrlParsed.search.substring(1) + "&host=" +
|
||||
itagUrlParsed.host;
|
||||
if (local) {
|
||||
if (
|
||||
Deno.env.get("ENCRYPT_QUERY_PARAMS") ||
|
||||
konfigStore.get("server.encrypt_query_params") as boolean
|
||||
) {
|
||||
queryParams = "enc=yes&data=" + encryptQuery(
|
||||
queryParams,
|
||||
konfigStore,
|
||||
);
|
||||
}
|
||||
urlToRedirect = (
|
||||
Deno.env.get("EXTERNAL_VIDEOPLAYBACK_PROXY") ||
|
||||
(konfigStore.get(
|
||||
"networking.external_videoplayback_proxy",
|
||||
) as string ?? "")
|
||||
) + (itagUrlParsed.pathname + "?" + queryParams) as string;
|
||||
urlToRedirect = (konfigStore.get(
|
||||
"networking.external_videoplayback_proxy",
|
||||
) as string ?? "") +
|
||||
itagUrlParsed.pathname + itagUrlParsed.search +
|
||||
"&host=" + itagUrlParsed.host;
|
||||
}
|
||||
return c.redirect(urlToRedirect);
|
||||
}
|
||||
|
|
|
@ -1,46 +1,16 @@
|
|||
import { Hono } from "hono";
|
||||
import { Counter, Gauge, Registry, } from "prom-client";
|
||||
// import { kv } from "../lib/helpers/youtubePlayerHandling.ts";
|
||||
import { cachedEntries, register } from "./index.ts";
|
||||
import { kv } from "../lib/helpers/youtubePlayerHandling.ts";
|
||||
|
||||
const metrics = new Hono();
|
||||
const METRICS_PREFIX = "invidious_companion_";
|
||||
export const register = new Registry();
|
||||
|
||||
function createCounter(name: string, help?: string): Counter {
|
||||
return new Counter({
|
||||
name: `${METRICS_PREFIX}${name}`,
|
||||
help: help || "TODO",
|
||||
registers: [register],
|
||||
});
|
||||
}
|
||||
|
||||
export const cachedEntries = new Gauge({
|
||||
name: `${METRICS_PREFIX}cached_entries`,
|
||||
help: "TODO",
|
||||
registers: [register],
|
||||
});
|
||||
|
||||
export const externalTokenGeneratorFail = createCounter(
|
||||
"externaltokengenerator_fail",
|
||||
);
|
||||
export const subreasonProtectCommunity = createCounter(
|
||||
"subreason_protect_community",
|
||||
);
|
||||
export const poTokenFail = createCounter("potoken_fail");
|
||||
export const reasonBot = createCounter("reason_bot", "TODO");
|
||||
export const videoUnavailable = createCounter("video_unavailable", "TODO");
|
||||
export const videoRestricted = createCounter("video_restricted", "TODO");
|
||||
export const failedRequests = createCounter("failed_requests", "TODO");
|
||||
export const successfulRequests = createCounter("successful_requests", "TODO");
|
||||
export const retryCount = createCounter("retry_count", "TODO");
|
||||
|
||||
metrics.get("/", async () => {
|
||||
// let i = 0;
|
||||
// const entries = kv.list({ prefix: ["video_cache"] });
|
||||
// for await (const _ of entries) {
|
||||
// i += 1;
|
||||
// }
|
||||
// cachedEntries.set(i);
|
||||
let i = 0;
|
||||
const entries = kv.list({ prefix: ["video_cache"] });
|
||||
for await (const _ of entries) {
|
||||
i += 1;
|
||||
}
|
||||
cachedEntries.set(i);
|
||||
return new Response(await register.metrics(), {
|
||||
headers: { "Content-Type": "text/plain" },
|
||||
});
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import { Hono } from "hono";
|
||||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import { HTTPException } from "hono/http-exception";
|
||||
import { decryptQuery } from "../lib/helpers/encrypter.ts";
|
||||
let getFetchClientLocation = "getFetchClient";
|
||||
if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
|
||||
if (Deno.env.has("DENO_COMPILED")) {
|
||||
|
@ -17,21 +17,9 @@ const { getFetchClient } = await import(getFetchClientLocation);
|
|||
const videoPlaybackProxy = new Hono();
|
||||
|
||||
videoPlaybackProxy.get("/", async (c) => {
|
||||
const konfigStore = c.get("konfigStore");
|
||||
let host, client, expire, urlReq, queryParams;
|
||||
|
||||
if (c.req.query("enc") === "yes") {
|
||||
const { data } = c.req.query();
|
||||
const unencryptedQueryParams = decryptQuery(data, konfigStore);
|
||||
queryParams = new URLSearchParams(unencryptedQueryParams);
|
||||
host = queryParams.get("host");
|
||||
client = queryParams.get("c");
|
||||
expire = queryParams.get("expire");
|
||||
} else {
|
||||
urlReq = new URL(c.req.url);
|
||||
queryParams = new URLSearchParams(urlReq.search);
|
||||
({ host, c: client, expire } = c.req.query());
|
||||
}
|
||||
const { host, c: client, expire } = c.req.query();
|
||||
const rangeHeader = c.req.header("range") as string | undefined;
|
||||
const urlReq = new URL(c.req.url);
|
||||
|
||||
if (host == undefined || !/[\w-]+.googlevideo.com/.test(host)) {
|
||||
throw new HTTPException(400, {
|
||||
|
@ -56,13 +44,18 @@ videoPlaybackProxy.get("/", async (c) => {
|
|||
});
|
||||
}
|
||||
|
||||
// @ts-ignore Do not understand how to fix this error.
|
||||
const konfigStore = await c.get("konfigStore") as Store<
|
||||
Record<string, unknown>
|
||||
>;
|
||||
|
||||
// deno-lint-ignore prefer-const
|
||||
let queryParams = new URLSearchParams(urlReq.search);
|
||||
queryParams.delete("host");
|
||||
const rangeHeader = c.req.header("range");
|
||||
const requestBytes = rangeHeader ? rangeHeader.split("=")[1] : null;
|
||||
if (requestBytes) {
|
||||
if (rangeHeader) {
|
||||
queryParams.append(
|
||||
"range",
|
||||
requestBytes,
|
||||
rangeHeader.split("=")[1],
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -109,7 +102,7 @@ videoPlaybackProxy.get("/", async (c) => {
|
|||
);
|
||||
}
|
||||
|
||||
const headersForResponse: Record<string, string> = {
|
||||
const headersForResponse = {
|
||||
"content-length": googlevideoResponse.headers.get("content-length") ||
|
||||
"",
|
||||
"access-control-allow-origin": "*",
|
||||
|
@ -119,14 +112,8 @@ videoPlaybackProxy.get("/", async (c) => {
|
|||
"last-modified": googlevideoResponse.headers.get("last-modified") || "",
|
||||
};
|
||||
|
||||
let responseStatus = googlevideoResponse.status;
|
||||
if (requestBytes && responseStatus == 200) {
|
||||
responseStatus = 206;
|
||||
headersForResponse["content-range"] = `bytes ${requestBytes}/*`;
|
||||
}
|
||||
|
||||
return new Response(googlevideoResponse.body, {
|
||||
status: responseStatus,
|
||||
status: googlevideoResponse.status,
|
||||
statusText: googlevideoResponse.statusText,
|
||||
headers: headersForResponse,
|
||||
});
|
||||
|
|
|
@ -1,13 +1,16 @@
|
|||
import { Hono } from "hono";
|
||||
import { youtubePlayerParsing } from "../../lib/helpers/youtubePlayerHandling.ts";
|
||||
import { HonoVariables } from "../../lib/types/HonoVariables.ts";
|
||||
import { Innertube } from "youtubei.js";
|
||||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import {
|
||||
reasonBot,
|
||||
subreasonProtectCommunity,
|
||||
videoRestricted,
|
||||
videoUnavailable,
|
||||
} from "metrics";
|
||||
} from "../index.ts";
|
||||
|
||||
const player = new Hono();
|
||||
const player = new Hono<{ Variables: HonoVariables }>();
|
||||
|
||||
const errors = [
|
||||
{
|
||||
|
@ -44,15 +47,17 @@ const errors = [
|
|||
|
||||
player.post("/player", async (c) => {
|
||||
const jsonReq = await c.req.json();
|
||||
const innertubeClient = c.get("innertubeClient");
|
||||
const konfigStore = c.get("konfigStore");
|
||||
const innertubeClient = await c.get("innertubeClient") as Innertube;
|
||||
// @ts-ignore Do not understand how to fix this error.
|
||||
const konfigStore = await c.get("konfigStore") as Store<
|
||||
Record<string, unknown>
|
||||
>;
|
||||
if (jsonReq.videoId) {
|
||||
const yt = await youtubePlayerParsing({
|
||||
const yt = await youtubePlayerParsing(
|
||||
innertubeClient,
|
||||
videoId: jsonReq.videoId,
|
||||
jsonReq.videoId,
|
||||
konfigStore,
|
||||
tokenMinter: c.get("tokenMinter"),
|
||||
});
|
||||
);
|
||||
errors.forEach((error) => {
|
||||
if (error.check(yt)) {
|
||||
error.action();
|
||||
|
|
Reference in a new issue