diff --git a/.gitignore b/.gitignore index 915d48a..8ad88ef 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,5 @@ Untitled*.ipynb invidious_companion test_things/ -config/local.toml \ No newline at end of file +config/local.toml +.cache/ \ No newline at end of file diff --git a/config/default.toml b/config/default.toml index b598e50..c1c50b3 100644 --- a/config/default.toml +++ b/config/default.toml @@ -14,4 +14,8 @@ enabled = true [jobs.youtube_session] po_token_enabled = true -frequency = "*/5 * * * *" \ No newline at end of file +frequency = "*/5 * * * *" + +[youtube_session] +oauth_enabled = false +cookies = "" \ No newline at end of file diff --git a/deno.json b/deno.json index 6472c20..e1545b4 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "tasks": { - "dev": "deno run --allow-net --allow-env --allow-read --allow-sys=hostname --watch src/main.ts", + "dev": "deno run --allow-net --allow-env --allow-read --allow-sys=hostname --allow-write=./,/tmp/youtubei.js --watch src/main.ts", "compile": "deno compile --output invidious_companion --allow-net --allow-env --allow-sys=hostname --allow-read src/main.ts" }, "imports": { @@ -8,6 +8,7 @@ "hono/logger": "jsr:@hono/hono@^4.6.5/logger", "hono/bearer-auth": "jsr:@hono/hono@^4.6.5/bearer-auth", "youtubei.js": "https://deno.land/x/youtubei@v11.0.1-deno/deno.ts", + "youtubei.js/endpoints": "https://deno.land/x/youtubei@v11.0.1-deno/deno/src/core/endpoints/index.ts", "jsdom": "https://esm.sh/jsdom@25.0.1", "bgutils": "https://esm.sh/bgutils-js@3.0.0", "estree": "npm:@types/estree", diff --git a/deno.lock b/deno.lock index 02b8b5c..ef4059e 100644 --- a/deno.lock +++ b/deno.lock @@ -7,6 +7,9 @@ "jsr:@std/fs": "jsr:@std/fs@1.0.4", "jsr:@std/path": "jsr:@std/path@1.0.6", "jsr:@std/path@^1.0.6": "jsr:@std/path@1.0.6", + "npm:@types/estree": "npm:@types/estree@1.0.6", + "npm:@types/estree@^1.0.6": "npm:@types/estree@1.0.6", + "npm:@types/node": "npm:@types/node@18.16.19", "npm:@willsoto/node-konfig-core@5.0.0": "npm:@willsoto/node-konfig-core@5.0.0", "npm:@willsoto/node-konfig-file@3.0.0": "npm:@willsoto/node-konfig-file@3.0.0_@willsoto+node-konfig-core@5.0.0", "npm:@willsoto/node-konfig-toml-parser@3.0.0": "npm:@willsoto/node-konfig-toml-parser@3.0.0_@willsoto+node-konfig-core@5.0.0", @@ -19,6 +22,7 @@ "@luanrt/jintr@3.0.2": { "integrity": "3b3bcf6af55f3c410fe575e11f1c5c1e738e5253df05169a5436848d7996d383", "dependencies": [ + "npm:@types/estree@^1.0.6", "npm:acorn@^8.8.0" ] }, @@ -33,6 +37,14 @@ } }, "npm": { + "@types/estree@1.0.6": { + "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", + "dependencies": {} + }, + "@types/node@18.16.19": { + "integrity": "sha512-IXl7o+R9iti9eBW4Wg2hx1xQDig183jj7YLn8F7udNceyfkbn1ZxmzZXuak20gR40D7pIkIY1kYGx5VIGbaHKA==", + "dependencies": {} + }, "@willsoto/node-konfig-core@5.0.0": { "integrity": "sha512-1AevWxJw/9oGz53YpabBSYhNTY1fsSIxiWd6ehSLCPnlgZ1ciJy6ZcwMpAb5L86dMjFHYwTa4Z8HiOP6iHyi+Q==", "dependencies": { @@ -71,6 +83,9 @@ } } }, + "redirects": { + "https://deno.land/x/youtubei/deno.ts": "https://deno.land/x/youtubei@v11.0.1-deno/deno.ts" + }, "remote": { "https://deno.land/std@0.159.0/encoding/ascii85.ts": "f2b9cb8da1a55b3f120d3de2e78ac993183a4fd00dfa9cb03b51cf3a75bc0baa", "https://deno.land/x/brotli@0.1.7/mod.ts": "08b913e51488b6e7fa181f2814b9ad087fdb5520041db0368f8156bfa45fd73e", @@ -652,10 +667,12 @@ "https://deno.land/x/youtubei@v11.0.1-deno/deno/src/platform/lib.ts": "fdac76db7d9f1c13039036f590270fe7860396a8afb24eac078122d91b4d6742", "https://deno.land/x/youtubei@v11.0.1-deno/deno/src/platform/polyfills/web-crypto.ts": "ae20ed00dea9eafca9ba590f4fa440299cbd57288add788c59cb19f3455ae6d1", "https://deno.land/x/youtubei@v11.0.1-deno/deno/src/types/Cache.ts": "06cd238bce7c9657055151587e36ee445e8236d54d27272124ced10ea7be0da4", + "https://deno.land/x/youtubei@v11.0.1-deno/deno/src/types/DashOptions.ts": "cf694c112ab97d778b3df735ddc76fd16fd5ae0d49943e2cc580f1f986f63da6", "https://deno.land/x/youtubei@v11.0.1-deno/deno/src/types/Endpoints.ts": "341ebfabf7099f88fc88b017635333646588d4d3001508026508d25457de1725", "https://deno.land/x/youtubei@v11.0.1-deno/deno/src/types/FormatUtils.ts": "9b53aefca649747856fc1ab89bfd98a63d245292e6fd4f3e3f3e9f2e5529c848", "https://deno.land/x/youtubei@v11.0.1-deno/deno/src/types/Misc.ts": "1acb401f52b4cd85726764a7708354c887f6fdc575cc0e4b8cfad42376931bac", "https://deno.land/x/youtubei@v11.0.1-deno/deno/src/types/PlatformShim.ts": "06f656f0d2bc20980ef77148455b662af10fe4b0e48d41566bf28e471eea4be1", + "https://deno.land/x/youtubei@v11.0.1-deno/deno/src/types/StreamingInfoOptions.ts": "8381b0723c3a96d42a8b28655209c13bbf6035846fb8cba5ad52d81b2cb560b4", "https://deno.land/x/youtubei@v11.0.1-deno/deno/src/types/index.ts": "0c861a21b15b858d47bb0dd45364354d8fb749ac3122ba586d140703fbba8e90", "https://deno.land/x/youtubei@v11.0.1-deno/deno/src/utils/Cache.ts": "fd90c88da32e9283adf065475a5cb4e680b5152a6bc06dd8a3dc9349358cab35", "https://deno.land/x/youtubei@v11.0.1-deno/deno/src/utils/Constants.ts": "6a1df0f81e0ca26ac31830dfb7afd03a9e77a43c4f69c42992f45d11d7fe5f7a", @@ -723,6 +740,7 @@ "workspace": { "dependencies": [ "jsr:@hono/hono@^4.6.5", + "npm:@types/estree", "npm:@willsoto/node-konfig-core@5.0.0", "npm:@willsoto/node-konfig-file@3.0.0", "npm:@willsoto/node-konfig-toml-parser@3.0.0" diff --git a/src/lib/helpers/getFetchClient.ts b/src/lib/helpers/getFetchClient.ts new file mode 100644 index 0000000..ae641b5 --- /dev/null +++ b/src/lib/helpers/getFetchClient.ts @@ -0,0 +1,33 @@ +import { Store } from "@willsoto/node-konfig-core"; + +export const getFetchClient = (konfigStore: Store): { + (input: RequestInfo | URL, init?: RequestInit): Promise; + (input: Request | URL | string, init?: RequestInit & { + client: Deno.HttpClient; + }): Promise; +} => { + if (konfigStore.get("networking.proxy")) { + return async ( + input: RequestInfo | URL, + init?: RequestInit, + ) => { + const client = Deno.createHttpClient({ + proxy: { + url: konfigStore.get("networking.proxy") as string, + }, + }); + const fetchRes = await fetch(input, { + client, + headers: init?.headers, + method: init?.method, + body: init?.body, + }); + return new Response(fetchRes.body, { + status: fetchRes.status, + headers: fetchRes.headers, + }); + }; + } + + return globalThis.fetch; +}; diff --git a/src/lib/helpers/konfigLoader.ts b/src/lib/helpers/konfigLoader.ts index c41b242..91f7a20 100644 --- a/src/lib/helpers/konfigLoader.ts +++ b/src/lib/helpers/konfigLoader.ts @@ -18,6 +18,7 @@ export const konfigLoader = async (): Promise< }; if (existsSync(pathJoin(Deno.cwd(), "config/local.toml"))) { + console.log("[INFO] Using custom settings local file.") konfigFilesToLoad.files.push({ path: pathJoin(Deno.cwd(), "config/local.toml"), parser: new TOMLParser(), diff --git a/src/lib/helpers/youtubePlayerHandling.ts b/src/lib/helpers/youtubePlayerHandling.ts index 586e9bc..b157f2d 100644 --- a/src/lib/helpers/youtubePlayerHandling.ts +++ b/src/lib/helpers/youtubePlayerHandling.ts @@ -10,15 +10,18 @@ export const youtubePlayerParsing = async ( videoId: string, konfigStore: Store, ): Promise => { + const cacheEnabled = konfigStore.get("cache.enabled"); + const videoCached = (await kv.get(["video_cache", videoId])) .value as Uint8Array; - if (videoCached != null) { + if (videoCached != null && cacheEnabled == true) { return JSON.parse(new TextDecoder().decode(decompress(videoCached))); } else { const youtubePlayerResponse = await youtubePlayerReq( innertubeClient, videoId, + konfigStore ); const videoData = youtubePlayerResponse.data; @@ -34,6 +37,7 @@ export const youtubePlayerParsing = async ( if (streamingData && videoData && videoData.streamingData) { const ecatcherServiceTracking = videoData.responseContext?.serviceTrackingParams.find(o => o.service === 'ECATCHER'); const clientNameUsed = ecatcherServiceTracking?.params?.find(o => o.key === 'client.name'); + // no need to decipher on IOS nor ANDROID if (!clientNameUsed?.value.includes("IOS") && !clientNameUsed?.value.includes("ANDROID")) { for (const [index, format] of streamingData.formats.entries()) { videoData.streamingData.formats[index].url = format.decipher( @@ -89,7 +93,7 @@ export const youtubePlayerParsing = async ( }, }))(videoData); - if (konfigStore.get("cache.enabled") == true && videoData.playabilityStatus?.status == "OK") { + if (cacheEnabled == true && videoData.playabilityStatus?.status == "OK") { (async () => { await kv.set( ["video_cache", videoId], diff --git a/src/lib/helpers/youtubePlayerReq.ts b/src/lib/helpers/youtubePlayerReq.ts index aedce90..b3f6617 100644 --- a/src/lib/helpers/youtubePlayerReq.ts +++ b/src/lib/helpers/youtubePlayerReq.ts @@ -1,7 +1,23 @@ import { Innertube, ApiResponse } from "youtubei.js"; +import { PlayerEndpoint } from "youtubei.js/endpoints"; +import { Store } from "@willsoto/node-konfig-core"; -export const youtubePlayerReq = async (innertubeClient: Innertube, videoId: string): Promise => { - return await innertubeClient.actions.execute("/player", { - videoId: videoId, - }); +export const youtubePlayerReq = async (innertubeClient: Innertube, videoId: string, konfigStore: Store): Promise => { + const innertubeClientOauthEnabled = konfigStore.get( + "youtube_session.oauth_enabled", + ) as boolean; + + let innertubeClientUsed = "WEB"; + if (innertubeClientOauthEnabled) + innertubeClientUsed = "TV"; + + return await innertubeClient.actions.execute( + PlayerEndpoint.PATH, PlayerEndpoint.build({ + video_id: videoId, + // @ts-ignore Unable to import type InnerTubeClient + client: innertubeClientUsed, + sts: innertubeClient.session.player?.sts, + po_token: innertubeClient.session.po_token + }) + ); }; diff --git a/src/lib/jobs/potoken.ts b/src/lib/jobs/potoken.ts index 22f5729..aef0090 100644 --- a/src/lib/jobs/potoken.ts +++ b/src/lib/jobs/potoken.ts @@ -3,6 +3,7 @@ import type { BgConfig } from "bgutils"; import { JSDOM } from "jsdom"; import { Innertube, UniversalCache } from "youtubei.js"; import { Store } from "@willsoto/node-konfig-core"; +import { getFetchClient } from "../helpers/getFetchClient.ts"; // Adapted from https://github.com/LuanRT/BgUtils/blob/main/examples/node/index.ts export const poTokenGenerate = async ( @@ -55,37 +56,12 @@ export const poTokenGenerate = async ( bgConfig, }); - await BG.PoToken.generatePlaceholder(visitorData); - - let fetchMethod = fetch; - - if (konfigStore.get("networking.proxy")) { - fetchMethod = async ( - input: RequestInfo | URL, - init?: RequestInit, - ) => { - const client = Deno.createHttpClient({ - proxy: { - url: konfigStore.get("networking.proxy") as string, - }, - }); - const fetchRes = await fetch(input, { - client, - headers: init?.headers, - method: init?.method, - body: init?.body, - }); - return new Response(fetchRes.body, { - status: fetchRes.status, - headers: fetchRes.headers, - }); - }; - } + await BG.PoToken.generatePlaceholder(visitorData);; return (await Innertube.create({ po_token: poTokenResult.poToken, visitor_data: visitorData, - fetch: fetchMethod, + fetch: getFetchClient(konfigStore), cache: new UniversalCache(true), generate_session_locally: true, })); diff --git a/src/main.ts b/src/main.ts index 44c8969..18ae305 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,26 +1,83 @@ import { Hono } from "hono"; import { routes } from "./routes/index.ts"; -import { Innertube } from "youtubei.js"; +import { Innertube, UniversalCache } from "youtubei.js"; import { poTokenGenerate } from "./lib/jobs/potoken.ts"; import { konfigLoader } from "./lib/helpers/konfigLoader.ts"; +import { getFetchClient } from "./lib/helpers/getFetchClient.ts"; const app = new Hono(); const konfigStore = await konfigLoader(); let innertubeClient: Innertube; +let innertubeClientUniversalCache = true; +let innertubeClientFetchPlayer = true; +const innertubeClientOauthEnabled = konfigStore.get( + "youtube_session.oauth_enabled", +) as boolean; +const innertubeClientJobPoTokenEnabled = konfigStore.get( + "jobs.youtube_session.po_token_enabled", +) as boolean; +const innertubeClientCookies = konfigStore.get( + "jobs.youtube_session.cookies", +) as string; -if (konfigStore.get("jobs.youtube_session.enabled") as boolean) { - innertubeClient = await Innertube.create({ retrieve_player: false }); - innertubeClient = await poTokenGenerate(innertubeClient, konfigStore); - Deno.cron("regenerate poToken", konfigStore.get("jobs.youtube_session.frequency") as string, async () => { +if (!innertubeClientOauthEnabled) { + if (innertubeClientJobPoTokenEnabled) { + console.log("[INFO] job po_token is active."); + // Don't fetch fetch player yet for po_token + innertubeClientFetchPlayer = false; + } else if (!innertubeClientJobPoTokenEnabled) { + console.log("[INFO] job po_token is NOT active."); + } +} else if (innertubeClientOauthEnabled) { + // Can't use cache if using OAuth#cacheCredentials + innertubeClientUniversalCache = false; +} + +innertubeClient = await Innertube.create({ + cache: new UniversalCache(innertubeClientUniversalCache), + retrieve_player: innertubeClientFetchPlayer, + fetch: getFetchClient(konfigStore), + cookie: innertubeClientCookies || undefined +}); + +if (!innertubeClientOauthEnabled) { + if (innertubeClientOauthEnabled) { innertubeClient = await poTokenGenerate(innertubeClient, konfigStore); + } + Deno.cron( + "regenerate youtube session", + konfigStore.get("jobs.youtube_session.frequency") as string, + async () => { + if (innertubeClientOauthEnabled) { + innertubeClient = await poTokenGenerate(innertubeClient, konfigStore); + } else { + innertubeClient = await Innertube.create({ + cache: new UniversalCache(innertubeClientUniversalCache), + retrieve_player: innertubeClientFetchPlayer, + }); + } + }, + ); +} else if (innertubeClientOauthEnabled) { + // Fired when waiting for the user to authorize the sign in attempt. + innertubeClient.session.on('auth-pending', (data) => { + console.log(`Go to ${data.verification_url} in your browser and enter code ${data.user_code} to authenticate.`); }); -} else { - innertubeClient = await Innertube.create(); - Deno.cron("regenerate visitordata", konfigStore.get("jobs.youtube_session.frequency") as string, async () => { - innertubeClient = await Innertube.create(); + // Fired when authentication is successful. + innertubeClient.session.on("auth", () => { + console.log("[INFO] [OAUTH] Sign in successful!"); }); + // Fired when the access token expires. + innertubeClient.session.on("update-credentials", async () => { + console.log("[INFO] [OAUTH] Credentials updated."); + await innertubeClient.session.oauth.cacheCredentials(); + }); + + // Attempt to sign in and then cache the credentials + await innertubeClient.session.signIn(); + await innertubeClient.session.oauth.cacheCredentials(); } app.use("*", async (c, next) => {