initial push

This commit is contained in:
Emilien Devos 2024-10-20 02:02:55 +02:00
commit 9c2f27db20
12 changed files with 478 additions and 0 deletions

41
.gitignore vendored Normal file
View file

@ -0,0 +1,41 @@
*.orig
*.pyc
*.swp
.env
/.cargo_home/
/.idea/
/.vs/
/.vscode/
gclient_config.py_entries
/target/
/std/hash/_wasm/target
/tests/wpt/runner/manifest.json
/third_party/
/tests/napi/node_modules
/tests/napi/build
/tests/napi/third_party_tests/node_modules
# MacOS generated files
.DS_Store
.DS_Store?
# Flamegraphs
/flamebench*.svg
/flamegraph*.svg
# WPT generated cert files
/tests/wpt/runner/certs/index.txt*
/tests/wpt/runner/certs/serial*
/ext/websocket/autobahn/reports
# JUnit files produced by deno test --junit
junit.xml
# Jupyter files
.ipynb_checkpoints/
Untitled*.ipynb
invidious_companion
test_things/

13
README.md Normal file
View file

@ -0,0 +1,13 @@
# Invidious companion
Companion for Invidious which handle all the video stream retrieval from YouTube servers.
## Requirements
- [deno](https://docs.deno.com/runtime/)
## Run Locally
```
deno task dev
```

10
config/default.toml Normal file
View file

@ -0,0 +1,10 @@
[server]
port = 8282
host = "127.0.0.1"
hmac_key = "CHANGE_ME"
[cache]
enabled = true
[networking]
#proxy = ""

5
config/local.toml Normal file
View file

@ -0,0 +1,5 @@
[cache]
enabled = true
[networking]
#proxy = "http://127.0.0.1:8899"

19
deno.json Normal file
View file

@ -0,0 +1,19 @@
{
"tasks": {
"dev": "deno run --allow-net --allow-env --allow-read --allow-sys=hostname --watch src/main.ts",
"compile": "deno compile --output invidious_companion --allow-net --allow-env --allow-sys=hostname --allow-read src/main.ts"
},
"imports": {
"@std/assert": "jsr:@std/assert@1",
"hono": "jsr:@hono/hono@^4.6.5",
"hono/logger": "jsr:@hono/hono@^4.6.5/logger",
"hono/bearer-auth": "jsr:@hono/hono@^4.6.5/bearer-auth",
"youtubei.js": "npm:youtubei.js@10.5.0/web",
"jsdom": "https://esm.sh/jsdom@25.0.1",
"bgutils": "https://esm.sh/bgutils-js@3.0.0",
"node-config": "npm:config@3.3.12",
"toml": "npm:toml@3.0.0",
"youtubePlayerReq": "./src/lib/helpers/youtubePlayerReq.ts"
},
"unstable": ["cron", "kv", "http"]
}

134
deno.lock generated Normal file
View file

@ -0,0 +1,134 @@
{
"version": "3",
"packages": {
"specifiers": {
"jsr:@hono/hono@^4.6.5": "jsr:@hono/hono@4.6.5",
"npm:config@3.3.12": "npm:config@3.3.12",
"npm:toml@3.0.0": "npm:toml@3.0.0",
"npm:youtubei.js@10.5.0": "npm:youtubei.js@10.5.0"
},
"jsr": {
"@hono/hono@4.6.5": {
"integrity": "68efe4a0ab7c4fb082cb71aa894a25e1c6cfad6d124dc943471e6758a7a1bdee"
}
},
"npm": {
"@bufbuild/protobuf@2.2.0": {
"integrity": "sha512-+imAQkHf7U/Rwvu0wk1XWgsP3WnpCWmK7B48f0XqSNzgk64+grljTKC7pnO/xBiEMUziF7vKRfbBnOQhg126qQ==",
"dependencies": {}
},
"@fastify/busboy@2.1.1": {
"integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==",
"dependencies": {}
},
"acorn@8.13.0": {
"integrity": "sha512-8zSiw54Oxrdym50NlZ9sUusyO1Z1ZchgRLWRaK6c86XJFClyCgFKetdowBg5bKxyp/u+CDBJG4Mpp0m3HLZl9w==",
"dependencies": {}
},
"config@3.3.12": {
"integrity": "sha512-Vmx389R/QVM3foxqBzXO8t2tUikYZP64Q6vQxGrsMpREeJc/aWRnPRERXWsYzOHAumx/AOoILWe6nU3ZJL+6Sw==",
"dependencies": {
"json5": "json5@2.2.3"
}
},
"jintr@2.1.1": {
"integrity": "sha512-89cwX4ouogeDGOBsEVsVYsnWWvWjchmwXBB4kiBhmjOKw19FiOKhNhMhpxhTlK2ctl7DS+d/ethfmuBpzoNNgA==",
"dependencies": {
"acorn": "acorn@8.13.0"
}
},
"json5@2.2.3": {
"integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
"dependencies": {}
},
"toml@3.0.0": {
"integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==",
"dependencies": {}
},
"tslib@2.8.0": {
"integrity": "sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA==",
"dependencies": {}
},
"undici@5.28.4": {
"integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==",
"dependencies": {
"@fastify/busboy": "@fastify/busboy@2.1.1"
}
},
"youtubei.js@10.5.0": {
"integrity": "sha512-iyA+VF28c15tCCKH9ExM2RKC3zYiHzA/eixGlJ3vERANkuI+xYKzAZ4vtOhmyqwrAddu88R/DkzEsmpph5NWjg==",
"dependencies": {
"@bufbuild/protobuf": "@bufbuild/protobuf@2.2.0",
"jintr": "jintr@2.1.1",
"tslib": "tslib@2.8.0",
"undici": "undici@5.28.4"
}
}
}
},
"redirects": {
"https://deno.land/x/brotli/mod.ts": "https://deno.land/x/brotli@0.1.7/mod.ts"
},
"remote": {
"https://deno.land/std@0.159.0/encoding/ascii85.ts": "f2b9cb8da1a55b3f120d3de2e78ac993183a4fd00dfa9cb03b51cf3a75bc0baa",
"https://deno.land/x/brotli@0.1.7/mod.ts": "08b913e51488b6e7fa181f2814b9ad087fdb5520041db0368f8156bfa45fd73e",
"https://deno.land/x/brotli@0.1.7/wasm.js": "77771b89e89ec7ff6e3e0939a7fb4f9b166abec3504cec0532ad5c127d6f35d2",
"https://deno.land/x/lz4@v0.1.2/mod.ts": "4decfc1a3569d03fd1813bd39128b71c8f082850fe98ecfdde20025772916582",
"https://deno.land/x/lz4@v0.1.2/wasm.js": "b9c65605327ba273f0c76a6dc596ec534d4cda0f0225d7a94ebc606782319e46",
"https://esm.sh/bgutils-js@3.0.0": "cc8243ff36620e02845bb12565036f93e464066cb0ec21b94b44562373e236bd",
"https://esm.sh/jsdom@25.0.1": "74c4782b56ddcf7cd1079c1d5e31832f0adbbb4099a116e360b5e0523444832f",
"https://esm.sh/v135/agent-base@7.1.0/denonext/agent-base.mjs": "0cfda332cb4694510eaeaa42dc88eed2223a9e6bed7352727a15ff163ee6b285",
"https://esm.sh/v135/agent-base@7.1.1/denonext/agent-base.mjs": "e7f92e882f955036b3054644e3b01623bfe61065102ddfffb98099966dad628c",
"https://esm.sh/v135/bgutils-js@3.0.0/denonext/bgutils-js.mjs": "de89db45dde68ef5ada5a3041611cc2083db2bccbdfdafbd773e937d3536de4d",
"https://esm.sh/v135/bufferutil@4.0.8/denonext/bufferutil.mjs": "60a4618cbd1a5cb24935c55590b793d4ecb33862357d32e1d4614a0bbb90947f",
"https://esm.sh/v135/canvas@2.11.2/denonext/canvas.mjs": "4245b1d01d91b5e807b85e40e98efe28c93634260bd8cb5ac0da71c42098a1a4",
"https://esm.sh/v135/cssstyle@4.1.0/denonext/cssstyle.mjs": "efe3d039b97cde71be202193e728d3f23636eca5301df3a83d21527202c4bf38",
"https://esm.sh/v135/data-urls@5.0.0/denonext/data-urls.mjs": "0a38da21608a5cf482ce7f18e78b24277fc6a63e58df0f3ec210b2810026ada5",
"https://esm.sh/v135/debug@4.3.4/denonext/debug.mjs": "d2ebf776ea77aa7df1b4460eb2a4aab245a9d5b19f11fa1db25f756b350bae9d",
"https://esm.sh/v135/debug@4.3.5/denonext/debug.mjs": "630202cad075e6db68a6c0113dcb2cd3a9b2e0a1b63a497a82e76229ef55cb33",
"https://esm.sh/v135/decimal.js@10.4.3/denonext/decimal.mjs": "936c013da678f9d5160e9a3aa4dd9a458040fce5dc0afb582f0ce7aa6413d572",
"https://esm.sh/v135/entities@4.5.0/denonext/lib/decode.js": "7fea6d8bd725edbbf7ea05031d2ea1bbbc1166dc11e3345d541198dd2dc16f1e",
"https://esm.sh/v135/entities@4.5.0/denonext/lib/escape.js": "7ebdc622bf3618bab25db40da4a49e2b9d03f044745f125f0bc3359f2d060def",
"https://esm.sh/v135/form-data@4.0.0/denonext/form-data.mjs": "48e84ac3b590bc364839367938d7e48ca37615a0c66e56dcc7356c3172ec7790",
"https://esm.sh/v135/html-encoding-sniffer@4.0.0/denonext/html-encoding-sniffer.mjs": "0063fd0b31101a12f0247fadb3ec62588abf656c34d6fea04cec631e40407593",
"https://esm.sh/v135/http-proxy-agent@7.0.2/denonext/http-proxy-agent.mjs": "df21dcd8cdb45e363640a913a218d334ac0dcdc1adb238523d06e9e9a1dfb6cc",
"https://esm.sh/v135/https-proxy-agent@7.0.5/denonext/https-proxy-agent.mjs": "3ae696a782a0bfc2a549d7f4d26d41f76bb15e8085b11f068ca19bef25b0a8a6",
"https://esm.sh/v135/iconv-lite@0.6.3/denonext/iconv-lite.mjs": "768e37377191ab3c7414bbb15fce202a328c411da0429764984c79b8bc65abd4",
"https://esm.sh/v135/is-potential-custom-element-name@1.0.1/denonext/is-potential-custom-element-name.mjs": "de2781ef99795b662f43c0840c3dcfdc303f9e60a75e66924370f902133469ed",
"https://esm.sh/v135/jsdom@25.0.1/denonext/jsdom.mjs": "166355ac54ff09711e7c6e8182ea68a008045d4921a8ef67fca627bc4a2998cd",
"https://esm.sh/v135/ms@2.1.2/denonext/ms.mjs": "aa4dc45ba72554c5011168f8910cc646c37af53cfff1a15a4decced838b8eb14",
"https://esm.sh/v135/node-gyp-build@4.6.1/denonext/node-gyp-build.mjs": "5d28b312f145a6cb2ec0dbdd80a7d34c0e0e6b5dcada65411d8bcff6c8991cc6",
"https://esm.sh/v135/node-gyp-build@4.8.1/denonext/node-gyp-build.mjs": "cddfc39c5f2d6e228fb1cd8cc36a594d870470b01348f866a7fb4e6f3ed8c66d",
"https://esm.sh/v135/nwsapi@2.2.12/denonext/nwsapi.mjs": "87d0568c6575b019ee9522a4bae67d9dafd1d20e6c24720170cb622aba67a63b",
"https://esm.sh/v135/parse5@7.1.2/denonext/parse5.mjs": "35bb04ec36a1c25c8cd8137296d64d16fd523a8ad1c2b63c41ba867fcd455c36",
"https://esm.sh/v135/rrweb-cssom@0.7.1/denonext/rrweb-cssom.mjs": "2b52b1670d2b5c4b7f1b27bdb76fbabf4915c4bfc6197710a3c62a5ff4975e7e",
"https://esm.sh/v135/safer-buffer@2.1.2/denonext/safer-buffer.mjs": "ce0e787812c668ba082ad5b75958490c714b6e05836bd5b6013d9f75727c006f",
"https://esm.sh/v135/saxes@6.0.0/denonext/saxes.mjs": "c788baa838835a5122681fee10909b16677ef3574fffd610a0c980321a95302d",
"https://esm.sh/v135/symbol-tree@3.2.4/denonext/symbol-tree.mjs": "67199d1e47bd6e5b7d2715dd04d25658061c95fc4464f7d200b6aab9e439b5f4",
"https://esm.sh/v135/tldts-core@6.1.47/denonext/tldts-core.mjs": "1ec163f0c44c05ab278859568719445a467f99d71ff63fe873a696e45560cbf1",
"https://esm.sh/v135/tldts@6.1.47/denonext/tldts.mjs": "9d166ad2aa7f9753aac76569b46d48ccf1e846401c268d12b8f1b7bc92c5522b",
"https://esm.sh/v135/tough-cookie@5.0.0/denonext/tough-cookie.mjs": "13a12fd7e56bd78bc54df00af86674dbffecd4bf9995a46b51ef45da09431cb9",
"https://esm.sh/v135/tr46@5.0.0/denonext/tr46.mjs": "66ea6f0789e30702596b0c5d0c2c2ae3e511aab829bb5b696938f61cd309e0dd",
"https://esm.sh/v135/utf-8-validate@6.0.4/denonext/utf-8-validate.mjs": "ab4990b545a45f10f7711c69046ee3e9c5b732b9781937f922cefd3fc99d0e88",
"https://esm.sh/v135/w3c-xmlserializer@5.0.0/denonext/w3c-xmlserializer.mjs": "62d486ecdf81d34bf972d00aae02e3bff998aafcfd2bdef73060c171e64277ce",
"https://esm.sh/v135/webidl-conversions@7.0.0/denonext/webidl-conversions.mjs": "04e3e6917179380727c6f65cd16a5a89836fb5a104fe5524c10a0a697f88d552",
"https://esm.sh/v135/whatwg-encoding@3.1.1/denonext/whatwg-encoding.mjs": "ecb7f2fe9fe4686e801ffaf4fa924e5860cbc3d07594a89e639d5c2ed20ca067",
"https://esm.sh/v135/whatwg-mimetype@4.0.0/denonext/whatwg-mimetype.mjs": "52577070194b4b1ebc78e6e9b457a078ca28d6f8477457bce914da555d92a5bb",
"https://esm.sh/v135/whatwg-url@14.0.0/denonext/webidl2js-wrapper.js": "0cbda10a2d527e2144a35c70d997736e1bba16c9b9a547a59b848ebe21a4c9d0",
"https://esm.sh/v135/whatwg-url@14.0.0/denonext/whatwg-url.mjs": "bd2aa23a676f3cfb590dc03f3b1b9afbfa0f41806714a65880ecc75c01b1d6a6",
"https://esm.sh/v135/ws@8.18.0/denonext/ws.mjs": "b4c6f51c7c4d60d2a880889c9b07e00aa367adf20a8304a365cc5e09051a5004",
"https://esm.sh/v135/xml-name-validator@5.0.0/denonext/xml-name-validator.mjs": "a6e6944763a721d5fd93a1a8fef6a35eb49e929d63e208440848c6a2a8055dcd",
"https://esm.sh/v135/xmlchars@2.2.0/denonext/xml/1.0/ed5.js": "60f8f018eb1d79d69a41324155b7d9f52f1058b37060b28acc1dfc49446e549d",
"https://esm.sh/v135/xmlchars@2.2.0/denonext/xml/1.1/ed2.js": "ba7d1fe5694f62469c4b293a1fadad332c637cbcfbc74147a296475c2ff8ad3d",
"https://esm.sh/v135/xmlchars@2.2.0/denonext/xmlns/1.0/ed3.js": "929d15ffc72d56c8909f87e7df8288f060bda0256622e8e95c24f0decb06adc7"
},
"workspace": {
"dependencies": [
"jsr:@hono/hono@^4.6.5",
"jsr:@std/assert@1",
"npm:config@3.3.12",
"npm:toml@3.0.0",
"npm:youtubei.js@10.5.0"
]
}
}

View file

@ -0,0 +1,7 @@
import { Innertube } from "youtubei.js";
export const youtubePlayerReq = async (innertubeClient: Innertube, videoId: string) => {
return await innertubeClient.actions.execute("/player", {
videoId: videoId,
});
};

91
src/lib/jobs/potoken.ts Normal file
View file

@ -0,0 +1,91 @@
import { BG } from "bgutils";
import type { BgConfig } from "bgutils";
import { JSDOM } from "jsdom";
import { Innertube, UniversalCache } from "youtubei.js";
import Config from "node-config";
export const poTokenGenerate = async (
innertubeClient: Innertube,
config: Config,
) => {
const requestKey = "O43z0dpjhgX20SCx4KAo";
if (innertubeClient.session.po_token) {
innertubeClient = await Innertube.create({ retrieve_player: false });
}
const visitorData = innertubeClient.session.context.client.visitorData;
if (!visitorData) {
throw new Error("Could not get visitor data");
}
const dom = new JSDOM();
Object.assign(globalThis, {
window: dom.window,
document: dom.window.document,
});
const bgConfig: BgConfig = {
fetch: (input: string | URL | globalThis.Request, init?: RequestInit) =>
fetch(input, init),
globalObj: globalThis,
identifier: visitorData,
requestKey,
};
const bgChallenge = await BG.Challenge.create(bgConfig);
if (!bgChallenge) {
throw new Error("Could not get challenge");
}
const interpreterJavascript = bgChallenge.interpreterJavascript
.privateDoNotAccessOrElseSafeScriptWrappedValue;
if (interpreterJavascript) {
new Function(interpreterJavascript)();
} else throw new Error("Could not load VM");
const poTokenResult = await BG.PoToken.generate({
program: bgChallenge.program,
globalName: bgChallenge.globalName,
bgConfig,
});
await BG.PoToken.generatePlaceholder(visitorData);
let fetchMethod = fetch;
if (config.has("networking.proxy")) {
fetchMethod = async (
input: RequestInfo | URL,
init?: RequestInit,
) => {
const client = Deno.createHttpClient({
proxy: {
url: config.get("networking.proxy"),
},
});
const fetchRes = await fetch(input, {
client,
headers: init?.headers,
method: init?.method,
body: init?.body,
});
return new Response(fetchRes.body, {
status: fetchRes.status,
headers: fetchRes.headers,
});
};
}
return (await Innertube.create({
po_token: poTokenResult.poToken,
visitor_data: visitorData,
fetch: fetchMethod,
cache: new UniversalCache(true),
generate_session_locally: true,
}));
};

View file

@ -0,0 +1,5 @@
import { Innertube } from "youtubei.js";
export type HonoVariables = {
innertubeClient: Innertube;
};

29
src/main.ts Normal file
View file

@ -0,0 +1,29 @@
import { Hono } from "hono";
import { routes } from "./routes/index.ts";
import { Innertube } from "youtubei.js";
import { poTokenGenerate } from "./lib/jobs/potoken.ts";
import config from "node-config";
// deno-lint-ignore no-unused-vars
import toml from "toml";
const app = new Hono();
let innertubeClient = await Innertube.create({ retrieve_player: false });
innertubeClient = await poTokenGenerate(innertubeClient, config);
Deno.cron("regenerate poToken", "*/10 * * * *", async () => {
innertubeClient = await poTokenGenerate(innertubeClient, config);
});
app.use("*", async (c, next) => {
// @ts-ignore Do not understand how to fix this error.
c.set("innertubeClient", innertubeClient);
// @ts-ignore Do not understand how to fix this error.
c.set("config", config)
await next();
});
routes(app);
Deno.serve({ port: config.get("server.port"), hostname: config.get("server.host") }, app.fetch);

19
src/routes/index.ts Normal file
View file

@ -0,0 +1,19 @@
import { Hono } from "hono";
import { bearerAuth } from "hono/bearer-auth";
import { logger } from "hono/logger";
import config from 'node-config';
import youtube_route_player from "./youtube_routes/player.ts";
export const routes = (app: Hono) => {
app.use("*", logger());
app.use(
"/youtubei/v1/*",
bearerAuth({
token: config.get("server.hmac_key"),
}),
);
app.route("/youtubei/v1", youtube_route_player);
};

View file

@ -0,0 +1,105 @@
import { Hono } from "hono";
import { Innertube, YT } from "youtubei.js";
import { compress, decompress } from "https://deno.land/x/brotli/mod.ts";
import { HonoVariables } from "../../lib/types/HonoVariables.ts";
import { youtubePlayerReq } from "../../lib/helpers/youtubePlayerReq.ts";
import Config from "node-config";
const player = new Hono<{ Variables: HonoVariables }>();
const kv = await Deno.openKv();
player.post("/player", async (c) => {
const jsonReq = await c.req.json();
if (jsonReq.videoId) {
const reqVideoId = jsonReq.videoId;
const innertubeClient: Innertube = await c.get("innertubeClient");
// @ts-ignore Do not understand how to fix this error.
const config: Config = await c.get("config") as Config;
const videoCached = (await kv.get(["video_cache", reqVideoId]))
.value as Uint8Array;
if (videoCached != null) {
return c.json(
JSON.parse(new TextDecoder().decode(decompress(videoCached))),
);
} else {
const youtubePlayerResponse = await youtubePlayerReq(
innertubeClient,
reqVideoId,
);
const videoData = youtubePlayerResponse.data;
const video = new YT.VideoInfo(
[youtubePlayerResponse],
innertubeClient.actions,
"",
);
const streamingData = video.streaming_data;
if (streamingData && videoData && videoData.streamingData) {
streamingData.adaptive_formats;
for (const [index, format] of streamingData.formats.entries()) {
videoData.streamingData.formats[index].url = format.decipher(
innertubeClient.session.player,
);
if (
videoData.streamingData.formats[index].signatureCipher !== undefined
) {
delete videoData.streamingData.formats[index].signatureCipher;
}
}
for (
const [index, adaptive_format] of streamingData.adaptive_formats
.entries()
) {
videoData.streamingData.adaptiveFormats[index].url = adaptive_format
.decipher(
innertubeClient.session.player,
);
if (
videoData.streamingData.adaptiveFormats[index].signatureCipher !==
undefined
) {
delete videoData.streamingData.adaptiveFormats[index]
.signatureCipher;
}
}
}
const videoOnlyNecessaryInfo = ((
{
captions,
playabilityStatus,
storyboards,
streamingData,
videoDetails,
microformat,
},
) => ({
captions,
playabilityStatus,
storyboards,
streamingData,
videoDetails,
microformat,
}))(videoData);
if (config.get("cache.enabled") == true) {
(async () => {
await kv.set(
["video_cache", reqVideoId],
compress(
new TextEncoder().encode(JSON.stringify(videoOnlyNecessaryInfo)),
),
{
expireIn: 1000 * 60 * 60,
},
);
})();
}
return c.json(videoOnlyNecessaryInfo);
}
}
});
export default player;