support for Content PO Token using BGUtils, and PO token checks using real videos (#56)
* feat: initial support for Content PO Token using BGUtils * fix: add canvas * chore: remove unnecessary PO token replacement * chore: remove unnecessary canvas addition * fix: missing semi-colon in hono variables * chore: fix deno.lock * set enable_session_cache to false on all Innertube instances * comment and log about the botguard Not Implemented canvas issue * chore: adapt message about error Not implemented * chore: update deno.lock * fix: use passed fetch implementation consistently in potoken job * feat: add check to ensure a random video loads when generating PO token, retry if it fails * chore: update deno.lock * fix: replace youtubePlayerParsing arguments with destructured object and allow cache overrides * chore: deno fmt * chore: reuse cacheEnabled variable * chore: swap to ternary for clarity * chore: deno fmt * fix: update to new youtubePlayerParsing parameter shape --------- Co-authored-by: Emilien <4016501+unixfox@users.noreply.github.com>
This commit is contained in:
Alex Maras
GitHub
parent
865c22e1fd
commit
8cf1a58c3e
11 changed files with 266 additions and 76 deletions
|
|
@ -31,4 +31,4 @@
|
|||
"fmt": {
|
||||
"indentWidth": 4
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
80
deno.lock
generated
80
deno.lock
generated
|
|
@ -5,7 +5,11 @@
|
|||
"jsr:@luanrt/googlevideo@2": "2.0.0",
|
||||
"jsr:@luanrt/jintr@^3.2.1": "3.2.1",
|
||||
"jsr:@std/async@*": "1.0.10",
|
||||
"npm:@bufbuild/protobuf@2": "2.2.2",
|
||||
"jsr:@std/encoding@*": "1.0.7",
|
||||
"jsr:@std/fs@*": "1.0.13",
|
||||
"jsr:@std/path@*": "1.0.8",
|
||||
"jsr:@std/path@^1.0.8": "1.0.8",
|
||||
"npm:@bufbuild/protobuf@2": "2.2.3",
|
||||
"npm:@types/estree@^1.0.6": "1.0.6",
|
||||
"npm:@willsoto/node-konfig-core@5.0.0": "5.0.0",
|
||||
"npm:@willsoto/node-konfig-file@3.0.0": "3.0.0_@willsoto+node-konfig-core@5.0.0",
|
||||
|
|
@ -32,6 +36,18 @@
|
|||
},
|
||||
"@std/async@1.0.10": {
|
||||
"integrity": "2ff1b1c7d33d1416159989b0f69e59ec7ee8cb58510df01e454def2108b3dbec"
|
||||
},
|
||||
"@std/encoding@1.0.7": {
|
||||
"integrity": "f631247c1698fef289f2de9e2a33d571e46133b38d042905e3eac3715030a82d"
|
||||
},
|
||||
"@std/fs@1.0.13": {
|
||||
"integrity": "756d3ff0ade91c9e72b228e8012b6ff00c3d4a4ac9c642c4dac083536bf6c605",
|
||||
"dependencies": [
|
||||
"jsr:@std/path@^1.0.8"
|
||||
]
|
||||
},
|
||||
"@std/path@1.0.8": {
|
||||
"integrity": "548fa456bb6a04d3c1a1e7477986b6cffbce95102d0bb447c67c4ee70e0364be"
|
||||
}
|
||||
},
|
||||
"npm": {
|
||||
|
|
@ -45,8 +61,8 @@
|
|||
"lru-cache"
|
||||
]
|
||||
},
|
||||
"@bufbuild/protobuf@2.2.2": {
|
||||
"integrity": "sha512-UNtPCbrwrenpmrXuRwn9jYpPoweNXj8X5sMvYgsqYyaH8jQ6LfUJSk3dJLnBK+6sfYPrF4iAIo5sd5HQ+tg75A=="
|
||||
"@bufbuild/protobuf@2.2.3": {
|
||||
"integrity": "sha512-tFQoXHJdkEOSwj5tRIZSPNUuXK3RaR7T1nUrPgbYX1pUbvqqaaZAsfo+NXBPsz5rZMSKVFrgK1WL8Q/MSLvprg=="
|
||||
},
|
||||
"@csstools/color-helpers@5.0.2": {
|
||||
"integrity": "sha512-JqWH1vsgdGcw2RR6VliXXdA0/59LttzlU8UlRT/iUUsEeWfYq8I+K0yhihEUTTHLRm1EXvpsCx3083EU15ecsA=="
|
||||
|
|
@ -337,11 +353,11 @@
|
|||
"symbol-tree@3.2.4": {
|
||||
"integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="
|
||||
},
|
||||
"tldts-core@6.1.79": {
|
||||
"integrity": "sha512-HM+Ud/2oQuHt4I43Nvjc213Zji/z25NSH5OkJskJwHXNtYh9DTRlHMDFhms9dFMP7qyve/yVaXFIxmcJ7TdOjw=="
|
||||
"tldts-core@6.1.82": {
|
||||
"integrity": "sha512-Jabl32m21tt/d/PbDO88R43F8aY98Piiz6BVH9ShUlOAiiAELhEqwrAmBocjAqnCfoUeIsRU+h3IEzZd318F3w=="
|
||||
},
|
||||
"tldts@6.1.79": {
|
||||
"integrity": "sha512-wjlYwK8lC/WcywLWf3A7qbK07SexezXjTRVwuPWXHvcjD7MnpPS2RXY5rLO3g12a8CNc7Y7jQRQsV7XyuBZjig==",
|
||||
"tldts@6.1.82": {
|
||||
"integrity": "sha512-KCTjNL9F7j8MzxgfTgjT+v21oYH38OidFty7dH00maWANAI2IsLw2AnThtTJi9HKALHZKQQWnNebYheadacD+g==",
|
||||
"dependencies": [
|
||||
"tldts-core"
|
||||
]
|
||||
|
|
@ -349,8 +365,8 @@
|
|||
"toml@3.0.0": {
|
||||
"integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w=="
|
||||
},
|
||||
"tough-cookie@5.1.1": {
|
||||
"integrity": "sha512-Ek7HndSVkp10hmHP9V4qZO1u+pn1RU5sI0Fw+jCU3lyvuMZcgqsNgc6CmJJZyByK4Vm/qotGRJlfgAX8q+4JiA==",
|
||||
"tough-cookie@5.1.2": {
|
||||
"integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==",
|
||||
"dependencies": [
|
||||
"tldts"
|
||||
]
|
||||
|
|
@ -397,13 +413,49 @@
|
|||
}
|
||||
},
|
||||
"redirects": {
|
||||
"https://esm.sh/@types/estree@1.0.6": "https://esm.sh/v135/@types/estree@1.0.6/index.d.ts"
|
||||
"https://esm.sh/@types/estree@1.0.6": "https://esm.sh/@types/estree@1.0.6/index.d.ts"
|
||||
},
|
||||
"remote": {
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/wire": "66a21da06aeea8e11f70e77406abad253fdc5acc75f084f9f418ec5b18eb7ee0",
|
||||
"https://esm.sh/bgutils-js@3.1.0": "a7a150393362ebecd356652158e3adcb4ba594ec5f1677cedc06c7debd10b9f7",
|
||||
"https://esm.sh/v135/@bufbuild/protobuf@2.0.0/denonext/wire.js": "6b8dc119872ecdd4078d46c9a290504190a1b9f58e402033c605ffc2deffc017",
|
||||
"https://esm.sh/v135/bgutils-js@3.1.0/denonext/bgutils-js.mjs": "4e6ad19cfed424a689ad31cdd761a04b9cb60d4b94429fc65e9a19e561906040",
|
||||
"https://deno.land/std@0.159.0/encoding/ascii85.ts": "f2b9cb8da1a55b3f120d3de2e78ac993183a4fd00dfa9cb03b51cf3a75bc0baa",
|
||||
"https://deno.land/x/brotli@0.1.7/mod.ts": "08b913e51488b6e7fa181f2814b9ad087fdb5520041db0368f8156bfa45fd73e",
|
||||
"https://deno.land/x/brotli@0.1.7/wasm.js": "77771b89e89ec7ff6e3e0939a7fb4f9b166abec3504cec0532ad5c127d6f35d2",
|
||||
"https://deno.land/x/crypto@v0.10.1/aes.ts": "0f4e5af07514a07d56ec01b2186c0153f13d6e00c26fc4e70692996af6280e48",
|
||||
"https://deno.land/x/crypto@v0.10.1/block-modes.ts": "a7ef359649a2cf235451e80aed7a5fda612d92ca2b158bf2a724b32899f8e455",
|
||||
"https://deno.land/x/crypto@v0.10.1/src/aes/consts.ts": "582aeed7afda2fe3deac4a60c4a9f29c60a7fb66f56645f95fa0ddab49bde994",
|
||||
"https://deno.land/x/crypto@v0.10.1/src/aes/mod.ts": "883d1e48d033dc4491f3a336c07235c5c4cb0371972476b8cdeea5e94ad2efbe",
|
||||
"https://deno.land/x/crypto@v0.10.1/src/block-modes/base.ts": "9006474c676782602ede9ea16aa49e8d084fd5670f6050641715a3d3085e1ba5",
|
||||
"https://deno.land/x/crypto@v0.10.1/src/block-modes/cbc.ts": "ce1ae944dd1912febd1d69c26c3a4ba18caeac9544ac3c62abd8b2429842de19",
|
||||
"https://deno.land/x/crypto@v0.10.1/src/block-modes/cfb.ts": "a1de2d9b81c0333be6c8d005019db1f4b2956264f8f0f93e7e51e8059ad16147",
|
||||
"https://deno.land/x/crypto@v0.10.1/src/block-modes/ctr.ts": "06fd8e338dbda0a6a7fa49718a0fd247830759820e61646a6cf611ad69a9d464",
|
||||
"https://deno.land/x/crypto@v0.10.1/src/block-modes/ecb.ts": "c346d692f16f8efbcc041c25dd761ca223ef92e03bb05457908953bf261ba325",
|
||||
"https://deno.land/x/crypto@v0.10.1/src/block-modes/mod.ts": "bb7e3ddafa15b1ca024b4b5013b23d134410d01e90ce3d5be5489eb9d5f601c5",
|
||||
"https://deno.land/x/crypto@v0.10.1/src/block-modes/ofb.ts": "0f77075505853b4ba1a55b4edecb17323f8a1489456a3d3b74717565cccbf2ef",
|
||||
"https://deno.land/x/crypto@v0.10.1/src/utils/bytes.ts": "7ccf6cb2d747d9b9de04c9a2494999957c1e86fd4e14bc228aad25283323f5a0",
|
||||
"https://deno.land/x/crypto@v0.10.1/src/utils/padding.ts": "544c51a471a413b15940bf08b285bc6a5db27796ff3cf240564f42701aba01dc",
|
||||
"https://deno.land/x/lz4@v0.1.2/mod.ts": "4decfc1a3569d03fd1813bd39128b71c8f082850fe98ecfdde20025772916582",
|
||||
"https://deno.land/x/lz4@v0.1.2/wasm.js": "b9c65605327ba273f0c76a6dc596ec534d4cda0f0225d7a94ebc606782319e46",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/create.mjs": "d9fcaeb3dfeb517c4d59843d78d7b51b3ad91b911e35bc317dc29e97e4ba9e67",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/descriptors.mjs": "7e5dd304539ba1889dea236a3635983d22038ae19bd4952d1d9323f1622a9aaa",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/from-binary.mjs": "8619f1c113211c1bfde08795f2a8f30faeb6b48fefecf044dd12580bdd77e176",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/is-message.mjs": "b6435ef48053c60211391b011012c25056f8527a4b1a1201c128460adb60db29",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/proto-int64.mjs": "43ff71e1af9d45f2a6ad6170f4e4aa6c20f98d47a0a68be3b3f8363764ce1bd8",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/reflect/error.mjs": "e90d2f17fe9bf940b349915978ff389790b267313eafd016f8c80ed3633e2759",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/reflect/guard.mjs": "b7c13bcf7826d6dd26904e33abdc18e83896b2d9fda783d988257f62849e02cb",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/reflect/reflect-check.mjs": "03168260147a043ccd73d45d2e3c5e1adb37aaac3ba4834c5eca63d0f5c2fd1b",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/reflect/reflect.mjs": "a4daf06c0f06e91c1aac94566e7831006e35da21e2e85afe3f17c77458adc3c8",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/reflect/scalar.mjs": "3d4e9550eb227008f020a2b29cabe6dbf863fa17251a5a966163aec135f0314e",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/reflect/unsafe.mjs": "2e5be946c0a5f2fd57aa0631cd0edea8a8b836e6ca3d1f884175029de13612c1",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/to-binary.mjs": "28a330bdbb17e5d18957d164c9511c199615d9b13a9223d6c67260b0f1fdeab4",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/wire/base64-encoding.mjs": "6d5ac314fcfdc2af5529ffa8e319d4fbb5d6f18720a0eed171d012dfce3bb25f",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/wire/binary-encoding.mjs": "125088da118dcbcfde7fa3d11b87d8da30fc30418aa046684c348eea0f842251",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/wire/text-encoding.mjs": "da32b0e15885c8963c118c907927e8ee60fee57d21e3988bfd3aebf44afb1265",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/wire/text-format.mjs": "9c8e39da4c3cbc0618e93219f7def9356774198a005d3da92870aaa41a2eebd3",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/wire/varint.mjs": "148131cfbd4bebfa7aed5c4131a3c76c1a72ee95daaf3acb392ddf755e9a8bf5",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/dist/esm/wkt/wrappers.mjs": "2fdb6acdda91c53c238b51c5ea6ef8def5b4e9871f93c439ab835c0a9f1e6070",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/denonext/wire.mjs": "791ca43f39084f37fabf37bcd77c788861bce7cc3d0d8528dc61ec9559956f46",
|
||||
"https://esm.sh/@bufbuild/protobuf@2.0.0/wire": "eb1be07dad5823fc419cc9e6f62077a70962ce42facb1a5240b7d5c3674e852f",
|
||||
"https://esm.sh/bgutils-js@3.1.0": "0527a704a9b97ed3f1b8c9621f73bc7e3f4c8198f8d63e0a0662a29755958de4",
|
||||
"https://esm.sh/bgutils-js@3.1.0/denonext/bgutils-js.mjs": "89a104b72045a1dce0c041d1e3c83a5469db0df634d4f233670e6859f8d5cf89",
|
||||
"https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno.ts": "f913bfdb3c66b2330fa8b531bd1e291437b836c9fbf002b9ae044bf14e1f397c",
|
||||
"https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/package.json": "4e1d16c0f95a2b58dc6dba7e0fe969b800794a6040ae02116e14eae2444e4429",
|
||||
"https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/protos/generated/misc/common.ts": "7d6ca8d7cd7eafe1f8d5ddc11c440566c418aeaf2d8a03a72eced7466a691039",
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { ApiResponse, Innertube, YT } from "youtubei.js";
|
|||
import { generateRandomString } from "youtubei.js/Utils";
|
||||
import { compress, decompress } from "https://deno.land/x/brotli@0.1.7/mod.ts";
|
||||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import type { BG } from "bgutils";
|
||||
let youtubePlayerReqLocation = "youtubePlayerReq";
|
||||
if (Deno.env.get("YT_PLAYER_REQ_LOCATION")) {
|
||||
if (Deno.env.has("DENO_COMPILED")) {
|
||||
|
|
@ -17,23 +18,34 @@ const { youtubePlayerReq } = await import(youtubePlayerReqLocation);
|
|||
|
||||
const kv = await Deno.openKv();
|
||||
|
||||
export const youtubePlayerParsing = async (
|
||||
innertubeClient: Innertube,
|
||||
videoId: string,
|
||||
konfigStore: Store,
|
||||
): Promise<object> => {
|
||||
const cacheEnabled = konfigStore.get("cache.enabled");
|
||||
export const youtubePlayerParsing = async ({
|
||||
innertubeClient,
|
||||
videoId,
|
||||
konfigStore,
|
||||
tokenMinter,
|
||||
overrideCache = false,
|
||||
}: {
|
||||
innertubeClient: Innertube;
|
||||
videoId: string;
|
||||
konfigStore: Store;
|
||||
tokenMinter: BG.WebPoMinter;
|
||||
overrideCache?: boolean;
|
||||
}): Promise<object> => {
|
||||
const cacheEnabled = overrideCache
|
||||
? false
|
||||
: konfigStore.get("cache.enabled");
|
||||
|
||||
const videoCached = (await kv.get(["video_cache", videoId]))
|
||||
.value as Uint8Array;
|
||||
|
||||
if (videoCached != null && cacheEnabled == true) {
|
||||
if (videoCached != null && cacheEnabled) {
|
||||
return JSON.parse(new TextDecoder().decode(decompress(videoCached)));
|
||||
} else {
|
||||
const youtubePlayerResponse = await youtubePlayerReq(
|
||||
innertubeClient,
|
||||
videoId,
|
||||
konfigStore,
|
||||
tokenMinter,
|
||||
);
|
||||
const videoData = youtubePlayerResponse.data;
|
||||
|
||||
|
|
@ -135,9 +147,7 @@ export const youtubePlayerParsing = async (
|
|||
microformat,
|
||||
}))(videoData);
|
||||
|
||||
if (
|
||||
cacheEnabled == true && videoData.playabilityStatus?.status == "OK"
|
||||
) {
|
||||
if (cacheEnabled && videoData.playabilityStatus?.status == "OK") {
|
||||
(async () => {
|
||||
await kv.set(
|
||||
["video_cache", videoId],
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
import { ApiResponse, Innertube } from "youtubei.js";
|
||||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import NavigationEndpoint from "youtubei.js/NavigationEndpoint";
|
||||
import type { BG } from "bgutils";
|
||||
|
||||
export const youtubePlayerReq = async (
|
||||
innertubeClient: Innertube,
|
||||
videoId: string,
|
||||
konfigStore: Store,
|
||||
tokenMinter: BG.WebPoMinter,
|
||||
): Promise<ApiResponse> => {
|
||||
const innertubeClientOauthEnabled = konfigStore.get(
|
||||
"youtube_session.oauth_enabled",
|
||||
|
|
@ -20,7 +22,9 @@ export const youtubePlayerReq = async (
|
|||
watchEndpoint: { videoId: videoId },
|
||||
});
|
||||
|
||||
return await watch_endpoint.call(innertubeClient.actions, {
|
||||
const contentPoToken = await tokenMinter.mintAsWebsafeString(videoId);
|
||||
|
||||
return watch_endpoint.call(innertubeClient.actions, {
|
||||
playbackContext: {
|
||||
contentPlaybackContext: {
|
||||
vis: 0,
|
||||
|
|
@ -30,7 +34,7 @@ export const youtubePlayerReq = async (
|
|||
},
|
||||
},
|
||||
serviceIntegrityDimensions: {
|
||||
poToken: innertubeClient.session.po_token,
|
||||
poToken: contentPoToken,
|
||||
},
|
||||
client: innertubeClientUsed,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,8 +1,12 @@
|
|||
import { BG } from "bgutils";
|
||||
import type { BgConfig } from "bgutils";
|
||||
import { BG, buildURL, GOOG_API_KEY, USER_AGENT } from "bgutils";
|
||||
import type { WebPoSignalOutput } from "bgutils";
|
||||
import { JSDOM } from "jsdom";
|
||||
import { Innertube, UniversalCache } from "youtubei.js";
|
||||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import {
|
||||
youtubePlayerParsing,
|
||||
youtubeVideoInfo,
|
||||
} from "../helpers/youtubePlayerHandling.ts";
|
||||
let getFetchClientLocation = "getFetchClient";
|
||||
if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
|
||||
if (Deno.env.has("DENO_COMPILED")) {
|
||||
|
|
@ -21,59 +25,160 @@ export const poTokenGenerate = async (
|
|||
innertubeClient: Innertube,
|
||||
konfigStore: Store<Record<string, unknown>>,
|
||||
innertubeClientCache: UniversalCache,
|
||||
): Promise<Innertube> => {
|
||||
const requestKey = "O43z0dpjhgX20SCx4KAo";
|
||||
|
||||
): Promise<{ innertubeClient: Innertube; tokenMinter: BG.WebPoMinter }> => {
|
||||
if (innertubeClient.session.po_token) {
|
||||
innertubeClient = await Innertube.create({ retrieve_player: false });
|
||||
innertubeClient = await Innertube.create({
|
||||
enable_session_cache: false,
|
||||
user_agent: USER_AGENT,
|
||||
retrieve_player: false,
|
||||
});
|
||||
}
|
||||
|
||||
const fetchImpl = await getFetchClient(konfigStore);
|
||||
|
||||
const visitorData = innertubeClient.session.context.client.visitorData;
|
||||
|
||||
if (!visitorData) {
|
||||
throw new Error("Could not get visitor data");
|
||||
}
|
||||
|
||||
const dom = new JSDOM();
|
||||
const dom = new JSDOM(
|
||||
'<!DOCTYPE html><html lang="en"><head><title></title></head><body></body></html>',
|
||||
{
|
||||
url: "https://www.youtube.com/",
|
||||
referrer: "https://www.youtube.com/",
|
||||
userAgent: USER_AGENT,
|
||||
},
|
||||
);
|
||||
|
||||
Object.assign(globalThis, {
|
||||
window: dom.window,
|
||||
document: dom.window.document,
|
||||
location: dom.window.location,
|
||||
origin: dom.window.origin,
|
||||
});
|
||||
|
||||
const bgConfig: BgConfig = {
|
||||
fetch: getFetchClient(konfigStore),
|
||||
globalObj: globalThis,
|
||||
identifier: visitorData,
|
||||
requestKey,
|
||||
};
|
||||
if (!Reflect.has(globalThis, "navigator")) {
|
||||
Object.defineProperty(globalThis, "navigator", {
|
||||
value: dom.window.navigator,
|
||||
});
|
||||
}
|
||||
|
||||
const bgChallenge = await BG.Challenge.create(bgConfig);
|
||||
|
||||
if (!bgChallenge) {
|
||||
const challengeResponse = await innertubeClient.getAttestationChallenge(
|
||||
"ENGAGEMENT_TYPE_UNBOUND",
|
||||
);
|
||||
if (!challengeResponse.bg_challenge) {
|
||||
throw new Error("Could not get challenge");
|
||||
}
|
||||
|
||||
const interpreterJavascript = bgChallenge.interpreterJavascript
|
||||
.privateDoNotAccessOrElseSafeScriptWrappedValue;
|
||||
const interpreterUrl = challengeResponse.bg_challenge.interpreter_url
|
||||
.private_do_not_access_or_else_trusted_resource_url_wrapped_value;
|
||||
const bgScriptResponse = await fetchImpl(
|
||||
`http:${interpreterUrl}`,
|
||||
);
|
||||
const interpreterJavascript = await bgScriptResponse.text();
|
||||
|
||||
if (interpreterJavascript) {
|
||||
new Function(interpreterJavascript)();
|
||||
} else throw new Error("Could not load VM");
|
||||
|
||||
const poTokenResult = await BG.PoToken.generate({
|
||||
program: bgChallenge.program,
|
||||
globalName: bgChallenge.globalName,
|
||||
bgConfig,
|
||||
// Botguard currently surfaces a "Not implemented" error here, due to the environment
|
||||
// not having a valid Canvas API in JSDOM. At the time of writing, this doesn't cause
|
||||
// any issues as the Canvas check doesn't appear to be an enforced element of the checks
|
||||
console.log(
|
||||
'[INFO] the "Not implemented: HTMLCanvasElement.prototype.getContext" error is normal. Please do not open a bug report about it.',
|
||||
);
|
||||
const botguard = await BG.BotGuardClient.create({
|
||||
program: challengeResponse.bg_challenge.program,
|
||||
globalName: challengeResponse.bg_challenge.global_name,
|
||||
globalObj: globalThis,
|
||||
});
|
||||
|
||||
await BG.PoToken.generatePlaceholder(visitorData);
|
||||
const webPoSignalOutput: WebPoSignalOutput = [];
|
||||
const botguardResponse = await botguard.snapshot({ webPoSignalOutput });
|
||||
const requestKey = "O43z0dpjhgX20SCx4KAo";
|
||||
|
||||
return (await Innertube.create({
|
||||
po_token: poTokenResult.poToken,
|
||||
const integrityTokenResponse = await fetchImpl(
|
||||
buildURL("GenerateIT", true),
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json+protobuf",
|
||||
"x-goog-api-key": GOOG_API_KEY,
|
||||
"x-user-agent": "grpc-web-javascript/0.1",
|
||||
"user-agent": USER_AGENT,
|
||||
},
|
||||
body: JSON.stringify([requestKey, botguardResponse]),
|
||||
},
|
||||
);
|
||||
|
||||
const response = await integrityTokenResponse.json() as unknown[];
|
||||
|
||||
if (typeof response[0] !== "string") {
|
||||
throw new Error("Could not get integrity token");
|
||||
}
|
||||
|
||||
const integrityTokenBasedMinter = await BG.WebPoMinter.create({
|
||||
integrityToken: response[0],
|
||||
}, webPoSignalOutput);
|
||||
|
||||
const sessionPoToken = await integrityTokenBasedMinter.mintAsWebsafeString(
|
||||
visitorData,
|
||||
);
|
||||
|
||||
const instantiatedInnertubeClient = await Innertube.create({
|
||||
enable_session_cache: false,
|
||||
po_token: sessionPoToken,
|
||||
visitor_data: visitorData,
|
||||
fetch: getFetchClient(konfigStore),
|
||||
cache: innertubeClientCache,
|
||||
generate_session_locally: true,
|
||||
}));
|
||||
});
|
||||
|
||||
try {
|
||||
const feed = await instantiatedInnertubeClient.getTrending();
|
||||
// get all videos and shuffle them randomly to avoid using the same trending video over and over
|
||||
const videos = feed.videos
|
||||
.filter((video) => video.type === "Video")
|
||||
.map((value) => ({ value, sort: Math.random() }))
|
||||
.sort((a, b) => a.sort - b.sort)
|
||||
.map(({ value }) => value);
|
||||
|
||||
const video = videos.find((video) => "id" in video);
|
||||
if (!video) {
|
||||
throw new Error("no videos with id found in trending");
|
||||
}
|
||||
|
||||
const youtubePlayerResponseJson = await youtubePlayerParsing({
|
||||
innertubeClient: instantiatedInnertubeClient,
|
||||
videoId: video.id,
|
||||
konfigStore,
|
||||
tokenMinter: integrityTokenBasedMinter,
|
||||
overrideCache: true,
|
||||
});
|
||||
const videoInfo = youtubeVideoInfo(
|
||||
instantiatedInnertubeClient,
|
||||
youtubePlayerResponseJson,
|
||||
);
|
||||
const validFormat = videoInfo.streaming_data?.adaptive_formats[0];
|
||||
if (!validFormat) {
|
||||
throw new Error(
|
||||
"failed to find valid video with adaptive format to check token against",
|
||||
);
|
||||
}
|
||||
const result = await fetchImpl(validFormat?.url, { method: "HEAD" });
|
||||
if (result.status !== 200) {
|
||||
throw new Error(
|
||||
`did not get a 200 when checking video, got ${result.status} instead`,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
console.log("Failed to get valid PO token, will retry", { err });
|
||||
throw err;
|
||||
}
|
||||
|
||||
return {
|
||||
innertubeClient: instantiatedInnertubeClient,
|
||||
tokenMinter: integrityTokenBasedMinter,
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
import { Innertube } from "youtubei.js";
|
||||
import { BG } from "bgutils";
|
||||
import type { konfigLoader } from "../helpers/konfigLoader.ts";
|
||||
|
||||
export type HonoVariables = {
|
||||
innertubeClient: Innertube;
|
||||
konfigStore: Awaited<ReturnType<typeof konfigLoader>>;
|
||||
tokenMinter: BG.WebPoMinter;
|
||||
};
|
||||
|
|
|
|||
30
src/main.ts
30
src/main.ts
|
|
@ -2,8 +2,12 @@ import { Hono } from "hono";
|
|||
import { routes } from "./routes/index.ts";
|
||||
import { Innertube, UniversalCache } from "youtubei.js";
|
||||
import { poTokenGenerate } from "./lib/jobs/potoken.ts";
|
||||
import { USER_AGENT } from "bgutils";
|
||||
import { konfigLoader } from "./lib/helpers/konfigLoader.ts";
|
||||
import { retry } from "jsr:@std/async";
|
||||
import type { HonoVariables } from "./lib/types/HonoVariables.ts";
|
||||
import type { BG } from "bgutils";
|
||||
|
||||
let getFetchClientLocation = "getFetchClient";
|
||||
if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) {
|
||||
if (Deno.env.has("DENO_COMPILED")) {
|
||||
|
|
@ -23,6 +27,7 @@ declare module "hono" {
|
|||
const app = new Hono();
|
||||
const konfigStore = await konfigLoader();
|
||||
|
||||
let tokenMinter: BG.WebPoMinter;
|
||||
let innertubeClient: Innertube;
|
||||
let innertubeClientFetchPlayer = true;
|
||||
const innertubeClientOauthEnabled = konfigStore.get(
|
||||
|
|
@ -55,34 +60,44 @@ if (!innertubeClientOauthEnabled) {
|
|||
}
|
||||
|
||||
innertubeClient = await Innertube.create({
|
||||
enable_session_cache: false,
|
||||
cache: innertubeClientCache,
|
||||
retrieve_player: innertubeClientFetchPlayer,
|
||||
fetch: getFetchClient(konfigStore),
|
||||
cookie: innertubeClientCookies || undefined,
|
||||
user_agent: USER_AGENT,
|
||||
});
|
||||
|
||||
if (!innertubeClientOauthEnabled) {
|
||||
if (innertubeClientJobPoTokenEnabled) {
|
||||
innertubeClient = await poTokenGenerate(
|
||||
innertubeClient,
|
||||
konfigStore,
|
||||
innertubeClientCache as UniversalCache,
|
||||
);
|
||||
({ innertubeClient, tokenMinter } = await retry(
|
||||
poTokenGenerate.bind(
|
||||
poTokenGenerate,
|
||||
innertubeClient,
|
||||
konfigStore,
|
||||
innertubeClientCache as UniversalCache,
|
||||
),
|
||||
{ minTimeout: 1_000, maxTimeout: 60_000, multiplier: 5, jitter: 0 },
|
||||
));
|
||||
}
|
||||
Deno.cron(
|
||||
"regenerate youtube session",
|
||||
konfigStore.get("jobs.youtube_session.frequency") as string,
|
||||
{ backoffSchedule: [5_000, 15_000, 60_000, 180_000] },
|
||||
async () => {
|
||||
if (innertubeClientJobPoTokenEnabled) {
|
||||
innertubeClient = await poTokenGenerate(
|
||||
({ innertubeClient, tokenMinter } = await poTokenGenerate(
|
||||
innertubeClient,
|
||||
konfigStore,
|
||||
innertubeClientCache,
|
||||
);
|
||||
));
|
||||
} else {
|
||||
innertubeClient = await Innertube.create({
|
||||
enable_session_cache: false,
|
||||
cache: innertubeClientCache,
|
||||
fetch: getFetchClient(konfigStore),
|
||||
retrieve_player: innertubeClientFetchPlayer,
|
||||
user_agent: USER_AGENT,
|
||||
});
|
||||
}
|
||||
},
|
||||
|
|
@ -111,6 +126,7 @@ if (!innertubeClientOauthEnabled) {
|
|||
|
||||
app.use("*", async (c, next) => {
|
||||
c.set("innertubeClient", innertubeClient);
|
||||
c.set("tokenMinter", tokenMinter);
|
||||
c.set("konfigStore", konfigStore);
|
||||
await next();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import { Hono } from "hono";
|
||||
import type { HonoVariables } from "../../lib/types/HonoVariables.ts";
|
||||
import { Store } from "@willsoto/node-konfig-core";
|
||||
import { verifyRequest } from "../../lib/helpers/verifyRequest.ts";
|
||||
import {
|
||||
youtubePlayerParsing,
|
||||
|
|
@ -19,9 +18,7 @@ interface AvailableCaption {
|
|||
const captionsHandler = new Hono<{ Variables: HonoVariables }>();
|
||||
captionsHandler.get("/:videoId", async (c) => {
|
||||
const { videoId } = c.req.param();
|
||||
const konfigStore = await c.get("konfigStore") as Store<
|
||||
Record<string, unknown>
|
||||
>;
|
||||
const konfigStore = c.get("konfigStore");
|
||||
|
||||
const check = c.req.query("check");
|
||||
|
||||
|
|
@ -37,13 +34,14 @@ captionsHandler.get("/:videoId", async (c) => {
|
|||
}
|
||||
}
|
||||
|
||||
const innertubeClient = await c.get("innertubeClient");
|
||||
const innertubeClient = c.get("innertubeClient");
|
||||
|
||||
const youtubePlayerResponseJson = await youtubePlayerParsing(
|
||||
const youtubePlayerResponseJson = await youtubePlayerParsing({
|
||||
innertubeClient,
|
||||
videoId,
|
||||
konfigStore,
|
||||
);
|
||||
tokenMinter: c.get("tokenMinter"),
|
||||
});
|
||||
|
||||
const videoInfo = youtubeVideoInfo(
|
||||
innertubeClient,
|
||||
|
|
|
|||
|
|
@ -29,11 +29,12 @@ dashManifest.get("/:videoId", async (c) => {
|
|||
}
|
||||
}
|
||||
|
||||
const youtubePlayerResponseJson = await youtubePlayerParsing(
|
||||
const youtubePlayerResponseJson = await youtubePlayerParsing({
|
||||
innertubeClient,
|
||||
videoId,
|
||||
konfigStore,
|
||||
);
|
||||
tokenMinter: c.get("tokenMinter"),
|
||||
});
|
||||
const videoInfo = youtubeVideoInfo(
|
||||
innertubeClient,
|
||||
youtubePlayerResponseJson,
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ latestVersion.get("/", async (c) => {
|
|||
}
|
||||
|
||||
const innertubeClient = c.get("innertubeClient");
|
||||
const konfigStore = await c.get("konfigStore");
|
||||
const konfigStore = c.get("konfigStore");
|
||||
|
||||
if (konfigStore.get("server.verify_requests") && check == undefined) {
|
||||
throw new HTTPException(400, {
|
||||
|
|
@ -33,11 +33,12 @@ latestVersion.get("/", async (c) => {
|
|||
}
|
||||
}
|
||||
|
||||
const youtubePlayerResponseJson = await youtubePlayerParsing(
|
||||
const youtubePlayerResponseJson = await youtubePlayerParsing({
|
||||
innertubeClient,
|
||||
id,
|
||||
videoId: id,
|
||||
konfigStore,
|
||||
);
|
||||
tokenMinter: c.get("tokenMinter"),
|
||||
});
|
||||
const videoInfo = youtubeVideoInfo(
|
||||
innertubeClient,
|
||||
youtubePlayerResponseJson,
|
||||
|
|
|
|||
|
|
@ -9,11 +9,12 @@ player.post("/player", async (c) => {
|
|||
const konfigStore = c.get("konfigStore");
|
||||
if (jsonReq.videoId) {
|
||||
return c.json(
|
||||
await youtubePlayerParsing(
|
||||
await youtubePlayerParsing({
|
||||
innertubeClient,
|
||||
jsonReq.videoId,
|
||||
videoId: jsonReq.videoId,
|
||||
konfigStore,
|
||||
),
|
||||
tokenMinter: c.get("tokenMinter"),
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
Reference in a new issue