diff --git a/.github/workflows/deno-check.yaml b/.github/workflows/deno-check.yaml index e494a83..dab4b52 100644 --- a/.github/workflows/deno-check.yaml +++ b/.github/workflows/deno-check.yaml @@ -19,6 +19,9 @@ jobs: - name: Verify formatting run: deno fmt --check src/** + + - name: Verify typing + run: deno check src/** - name: Run linter run: deno lint diff --git a/Dockerfile b/Dockerfile index a413369..a8934b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,11 @@ RUN curl -fsSL https://github.com/krallin/tini/releases/download/v${TINI_VERSION --output /tini \ && chmod +x /tini +RUN arch=$(uname -m) && \ + curl -fsSL https://github.com/dmikusa/tiny-health-checker/releases/download/v0.36.0/thc-${arch}-unknown-linux-musl \ + --output /thc \ + && chmod +x /thc + RUN deno task compile # Stage for creating the non-privileged user @@ -29,12 +34,16 @@ RUN adduser -u 10001 -S appuser FROM gcr.io/distroless/cc COPY --from=builder /app/invidious_companion /app/ +COPY --from=builder /thc /thc COPY ./config/ /app/config/ COPY --from=builder /tini /tini ENV PORT=8282 \ HOST=0.0.0.0 +ENV THC_PORT=${PORT} \ + THC_PATH=/healthz + # Copy passwd file for the non-privileged user from the user-stage COPY --from=user-stage /etc/passwd /etc/passwd @@ -49,4 +58,4 @@ USER appuser ENTRYPOINT ["/tini", "--", "/app/invidious_companion"] -# HEALTHCHECK --interval=5s --timeout=5s --start-period=10s --retries=3 CMD ["/tini", "--", "/app/invidious_companion", "healthcheck"] +HEALTHCHECK --interval=5s --timeout=5s --start-period=10s --retries=5 CMD ["/thc"] diff --git a/config/default.toml b/config/default.toml index d91d0a5..acc7ba0 100644 --- a/config/default.toml +++ b/config/default.toml @@ -3,7 +3,6 @@ port = 8282 host = "127.0.0.1" # secret key needs to be 16 characters long or more secret_key = "CHANGE_ME" -base_url = "http://localhost:8282" verify_requests = false # max_dash_resolution = 1080 # encrypt_query_params = false diff --git a/deno.json b/deno.json index 133996c..7e2d00e 100644 --- a/deno.json +++ b/deno.json @@ -4,14 +4,15 @@ "compile": "deno compile --include ./src/lib/helpers/youtubePlayerReq.ts --include ./src/lib/helpers/getFetchClient.ts --output invidious_companion --allow-import=github.com:443,jsr.io:443,raw.githubusercontent.com:443,esm.sh:443,deno.land:443 --allow-net --allow-env --allow-read --allow-sys=hostname --allow-write=/var/tmp/youtubei.js --unsafely-ignore-certificate-errors src/main.ts" }, "imports": { - "hono": "jsr:@hono/hono@^4.6.5", - "hono/logger": "jsr:@hono/hono@^4.6.5/logger", - "hono/bearer-auth": "jsr:@hono/hono@^4.6.5/bearer-auth", + "hono": "jsr:@hono/hono@^4.7.2", + "hono/logger": "jsr:@hono/hono@^4.7.2/logger", + "hono/bearer-auth": "jsr:@hono/hono@^4.7.2/bearer-auth", "prom-client": "npm:prom-client@^15.1.3", "youtubei.js": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno.ts", "youtubei.js/Utils": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/utils/Utils.ts", "youtubei.js/NavigationEndpoint": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/parser/classes/NavigationEndpoint.ts", - "jsdom": "npm:jsdom@25.0.1", + "youtubei.js/PlayerCaptionsTracklist": "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/parser/classes/PlayerCaptionsTracklist.ts", + "jsdom": "npm:jsdom@26.0.0", "bgutils": "https://esm.sh/bgutils-js@3.1.0", "estree": "https://esm.sh/@types/estree@1.0.6", "@willsoto/node-konfig-core": "npm:@willsoto/node-konfig-core@5.0.0", @@ -19,15 +20,17 @@ "@willsoto/node-konfig-toml-parser": "npm:@willsoto/node-konfig-toml-parser@3.0.0", "youtubePlayerReq": "./src/lib/helpers/youtubePlayerReq.ts", "getFetchClient": "./src/lib/helpers/getFetchClient.ts", - "googlevideo": "npm:googlevideo@2.0.0", - "metrics": "./src/routes/metrics.ts" + "metrics": "./src/routes/metrics.ts", + "googlevideo": "jsr:@luanrt/googlevideo@^2.0.0", + "jsr:@luanrt/jintr": "jsr:@luanrt/jintr@^3.2.1" }, "unstable": [ "cron", "kv", - "http" + "http", + "temporal" ], "fmt": { "indentWidth": 4 } -} +} \ No newline at end of file diff --git a/deno.lock b/deno.lock index 7da98f6..9404a72 100644 --- a/deno.lock +++ b/deno.lock @@ -1,41 +1,38 @@ { "version": "4", "specifiers": { - "jsr:@hono/hono@^4.6.5": "4.6.19", - "jsr:@luanrt/jintr@*": "3.2.1", + "jsr:@hono/hono@^4.7.2": "4.7.2", + "jsr:@luanrt/jintr@^3.2.1": "3.2.1", "jsr:@std/async@*": "1.0.10", - "jsr:@std/encoding@*": "1.0.6", - "jsr:@std/fs@*": "1.0.10", + "jsr:@std/encoding@*": "1.0.7", + "jsr:@std/fs@*": "1.0.13", "jsr:@std/path@*": "1.0.8", "jsr:@std/path@^1.0.8": "1.0.8", - "npm:@types/estree@^1.0.6": "1.0.6", "npm:@willsoto/node-konfig-core@5.0.0": "5.0.0", "npm:@willsoto/node-konfig-file@3.0.0": "3.0.0_@willsoto+node-konfig-core@5.0.0", "npm:@willsoto/node-konfig-toml-parser@3.0.0": "3.0.0_@willsoto+node-konfig-core@5.0.0", "npm:acorn@^8.8.0": "8.14.0", - "npm:googlevideo@2.0.0": "2.0.0", - "npm:jsdom@25.0.1": "25.0.1", + "npm:jsdom@26.0.0": "26.0.0", "npm:prom-client@^15.1.3": "15.1.3" }, "jsr": { - "@hono/hono@4.6.19": { - "integrity": "5ba1bd0ef74449c0a647f029e29896776c30fb64aefbcef8724af4ce4846791b" + "@hono/hono@4.7.2": { + "integrity": "73466a24bd6eb3b527cde18e59ca5543890ddacb4312fc0bc7504d28a7e57a38" }, "@luanrt/jintr@3.2.1": { "integrity": "78acef6eb1c0e54303c14f77233a610dcc8e9da386f5e59c6167a2c5d8fdbe87", "dependencies": [ - "npm:@types/estree", "npm:acorn" ] }, "@std/async@1.0.10": { "integrity": "2ff1b1c7d33d1416159989b0f69e59ec7ee8cb58510df01e454def2108b3dbec" }, - "@std/encoding@1.0.6": { - "integrity": "ca87122c196e8831737d9547acf001766618e78cd8c33920776c7f5885546069" + "@std/encoding@1.0.7": { + "integrity": "f631247c1698fef289f2de9e2a33d571e46133b38d042905e3eac3715030a82d" }, - "@std/fs@1.0.10": { - "integrity": "bf041f9d7a0a460817f0421be8946d0e06011b3433e6c83a215628de5e3c7c2c", + "@std/fs@1.0.13": { + "integrity": "756d3ff0ade91c9e72b228e8012b6ff00c3d4a4ac9c642c4dac083536bf6c605", "dependencies": [ "jsr:@std/path@^1.0.8" ] @@ -55,21 +52,18 @@ "lru-cache" ] }, - "@bufbuild/protobuf@2.2.3": { - "integrity": "sha512-tFQoXHJdkEOSwj5tRIZSPNUuXK3RaR7T1nUrPgbYX1pUbvqqaaZAsfo+NXBPsz5rZMSKVFrgK1WL8Q/MSLvprg==" + "@csstools/color-helpers@5.0.2": { + "integrity": "sha512-JqWH1vsgdGcw2RR6VliXXdA0/59LttzlU8UlRT/iUUsEeWfYq8I+K0yhihEUTTHLRm1EXvpsCx3083EU15ecsA==" }, - "@csstools/color-helpers@5.0.1": { - "integrity": "sha512-MKtmkA0BX87PKaO1NFRTFH+UnkgnmySQOvNxJubsadusqPEC2aJ9MOQiMceZJJ6oitUl/i0L6u0M1IrmAOmgBA==" - }, - "@csstools/css-calc@2.1.1_@csstools+css-parser-algorithms@3.0.4__@csstools+css-tokenizer@3.0.3_@csstools+css-tokenizer@3.0.3": { - "integrity": "sha512-rL7kaUnTkL9K+Cvo2pnCieqNpTKgQzy5f+N+5Iuko9HAoasP+xgprVh7KN/MaJVvVL1l0EzQq2MoqBHKSrDrag==", + "@csstools/css-calc@2.1.2_@csstools+css-parser-algorithms@3.0.4__@csstools+css-tokenizer@3.0.3_@csstools+css-tokenizer@3.0.3": { + "integrity": "sha512-TklMyb3uBB28b5uQdxjReG4L80NxAqgrECqLZFQbyLekwwlcDDS8r3f07DKqeo8C4926Br0gf/ZDe17Zv4wIuw==", "dependencies": [ "@csstools/css-parser-algorithms", "@csstools/css-tokenizer" ] }, - "@csstools/css-color-parser@3.0.7_@csstools+css-parser-algorithms@3.0.4__@csstools+css-tokenizer@3.0.3_@csstools+css-tokenizer@3.0.3": { - "integrity": "sha512-nkMp2mTICw32uE5NN+EsJ4f5N+IGFeCFu4bGpiKgb2Pq/7J/MpyLBeQ5ry4KKtRFZaYs6sTmcMYrSRIyj5DFKA==", + "@csstools/css-color-parser@3.0.8_@csstools+css-parser-algorithms@3.0.4__@csstools+css-tokenizer@3.0.3_@csstools+css-tokenizer@3.0.3": { + "integrity": "sha512-pdwotQjCCnRPuNi06jFuP68cykU1f3ZWExLe/8MQ1LOs8Xq+fTkYgd+2V8mWUWMrOn9iS2HftPVaMZDaXzGbhQ==", "dependencies": [ "@csstools/color-helpers", "@csstools/css-calc", @@ -89,9 +83,6 @@ "@opentelemetry/api@1.9.0": { "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==" }, - "@types/estree@1.0.6": { - "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==" - }, "@willsoto/node-konfig-core@5.0.0": { "integrity": "sha512-1AevWxJw/9oGz53YpabBSYhNTY1fsSIxiWd6ehSLCPnlgZ1ciJy6ZcwMpAb5L86dMjFHYwTa4Z8HiOP6iHyi+Q==", "dependencies": [ @@ -124,6 +115,13 @@ "bintrees@1.0.2": { "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" }, + "call-bind-apply-helpers@1.0.2": { + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "dependencies": [ + "es-errors", + "function-bind" + ] + }, "cockatiel@2.0.2": { "integrity": "sha512-ehw7t3twohGiMTxARX0AcFiUxndXLhnIBWbnRnHtfde2jRywlPpPB/o3s9YSptXPj6tkOG0fzET4CUUx4GIpEg==" }, @@ -137,7 +135,7 @@ "integrity": "sha512-9+vem03dMXG7gDmZ62uqmRiMRNtinIZ9ZyuF6BdxzfOD+FdN5hretzynkn0ReS2DO2GSw76RWHs0UmJPI2zUjw==", "dependencies": [ "@asamuzakjp/css-color", - "rrweb-cssom@0.8.0" + "rrweb-cssom" ] }, "data-urls@5.0.0": { @@ -159,21 +157,88 @@ "delayed-stream@1.0.0": { "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, + "dunder-proto@1.0.1": { + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dependencies": [ + "call-bind-apply-helpers", + "es-errors", + "gopd" + ] + }, "entities@4.5.0": { "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==" }, - "form-data@4.0.1": { - "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", + "es-define-property@1.0.1": { + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==" + }, + "es-errors@1.3.0": { + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==" + }, + "es-object-atoms@1.1.1": { + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dependencies": [ + "es-errors" + ] + }, + "es-set-tostringtag@2.1.0": { + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dependencies": [ + "es-errors", + "get-intrinsic", + "has-tostringtag", + "hasown" + ] + }, + "form-data@4.0.2": { + "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", "dependencies": [ "asynckit", "combined-stream", + "es-set-tostringtag", "mime-types" ] }, - "googlevideo@2.0.0": { - "integrity": "sha512-OVlNWZ07TPIelaEII6mH9od+Cxljl7P4AzhEYVNN5d4FhFT9L5otpcLtgvraTE9u69KfVVw+L4pVeczArcD33w==", + "function-bind@1.1.2": { + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" + }, + "get-intrinsic@1.3.0": { + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", "dependencies": [ - "@bufbuild/protobuf" + "call-bind-apply-helpers", + "es-define-property", + "es-errors", + "es-object-atoms", + "function-bind", + "get-proto", + "gopd", + "has-symbols", + "hasown", + "math-intrinsics" + ] + }, + "get-proto@1.0.1": { + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dependencies": [ + "dunder-proto", + "es-object-atoms" + ] + }, + "gopd@1.2.0": { + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==" + }, + "has-symbols@1.1.0": { + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==" + }, + "has-tostringtag@1.0.2": { + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dependencies": [ + "has-symbols" + ] + }, + "hasown@2.0.2": { + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dependencies": [ + "function-bind" ] }, "html-encoding-sniffer@4.0.0": { @@ -205,8 +270,8 @@ "is-potential-custom-element-name@1.0.1": { "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==" }, - "jsdom@25.0.1": { - "integrity": "sha512-8i7LzZj7BF8uplX+ZyOlIz86V6TAsSs+np6m1kpW9u0JWi4z/1t+FzcK1aek+ybTnAC4KhBL4uXCNT0wcUIeCw==", + "jsdom@26.0.0": { + "integrity": "sha512-BZYDGVAIriBWTpIxYzrXjv3E/4u8+/pSG5bQdIYCbNCGOvsPkDQfTVLAIXAf9ETdCpduCVTkDe2NNZ8NIwUVzw==", "dependencies": [ "cssstyle", "data-urls", @@ -218,7 +283,7 @@ "is-potential-custom-element-name", "nwsapi", "parse5", - "rrweb-cssom@0.7.1", + "rrweb-cssom", "saxes", "symbol-tree", "tough-cookie", @@ -237,6 +302,9 @@ "lru-cache@10.4.3": { "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" }, + "math-intrinsics@1.1.0": { + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==" + }, "mime-db@1.52.0": { "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" }, @@ -268,9 +336,6 @@ "punycode@2.3.1": { "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==" }, - "rrweb-cssom@0.7.1": { - "integrity": "sha512-TrEMa7JGdVm0UThDJSx7ddw5nVm3UJS9o9CCIZ72B1vSyEZoziDqBYP3XIoi/12lKrJR8rE3jeFHMok2F/Mnsg==" - }, "rrweb-cssom@0.8.0": { "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==" }, @@ -292,11 +357,11 @@ "bintrees" ] }, - "tldts-core@6.1.75": { - "integrity": "sha512-AOvV5YYIAFFBfransBzSTyztkc3IMfz5Eq3YluaRiEu55nn43Fzaufx70UqEKYr8BoLCach4q8g/bg6e5+/aFw==" + "tldts-core@6.1.82": { + "integrity": "sha512-Jabl32m21tt/d/PbDO88R43F8aY98Piiz6BVH9ShUlOAiiAELhEqwrAmBocjAqnCfoUeIsRU+h3IEzZd318F3w==" }, - "tldts@6.1.75": { - "integrity": "sha512-+lFzEXhpl7JXgWYaXcB6DqTYXbUArvrWAE/5ioq/X3CdWLbDjpPP4XTrQBmEJ91y3xbe4Fkw7Lxv4P3GWeJaNg==", + "tldts@6.1.82": { + "integrity": "sha512-KCTjNL9F7j8MzxgfTgjT+v21oYH38OidFty7dH00maWANAI2IsLw2AnThtTJi9HKALHZKQQWnNebYheadacD+g==", "dependencies": [ "tldts-core" ] @@ -304,8 +369,8 @@ "toml@3.0.0": { "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==" }, - "tough-cookie@5.1.0": { - "integrity": "sha512-rvZUv+7MoBYTiDmFPBrhL7Ujx9Sk+q9wwm22x8c8T5IJaR+Wsyc7TNxbVxo84kZoRJZZMazowFLqpankBEQrGg==", + "tough-cookie@5.1.2": { + "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", "dependencies": [ "tldts" ] @@ -334,15 +399,15 @@ "whatwg-mimetype@4.0.0": { "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==" }, - "whatwg-url@14.1.0": { - "integrity": "sha512-jlf/foYIKywAt3x/XWKZ/3rz8OSJPiWktjmk891alJUEjiVxKX9LEO92qH3hv4aJ0mN3MWPvGMCy8jQi95xK4w==", + "whatwg-url@14.1.1": { + "integrity": "sha512-mDGf9diDad/giZ/Sm9Xi2YcyzaFpbdLpJPr+E9fSkyQ7KpQD4SdFcugkRQYzhmfI4KeV4Qpnn2sKPdo+kmsgRQ==", "dependencies": [ "tr46", "webidl-conversions" ] }, - "ws@8.18.0": { - "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==" + "ws@8.18.1": { + "integrity": "sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==" }, "xml-name-validator@5.0.0": { "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==" @@ -351,9 +416,6 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" } }, - "redirects": { - "https://esm.sh/@types/estree@1.0.6": "https://esm.sh/@types/estree@1.0.6/index.d.ts" - }, "remote": { "https://deno.land/std@0.159.0/encoding/ascii85.ts": "f2b9cb8da1a55b3f120d3de2e78ac993183a4fd00dfa9cb03b51cf3a75bc0baa", "https://deno.land/x/brotli@0.1.7/mod.ts": "08b913e51488b6e7fa181f2814b9ad087fdb5520041db0368f8156bfa45fd73e", @@ -1013,11 +1075,9 @@ "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/platform/lib.ts": "fdac76db7d9f1c13039036f590270fe7860396a8afb24eac078122d91b4d6742", "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/platform/polyfills/web-crypto.ts": "ae20ed00dea9eafca9ba590f4fa440299cbd57288add788c59cb19f3455ae6d1", "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/types/Cache.ts": "06cd238bce7c9657055151587e36ee445e8236d54d27272124ced10ea7be0da4", - "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/types/DashOptions.ts": "cf694c112ab97d778b3df735ddc76fd16fd5ae0d49943e2cc580f1f986f63da6", "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/types/FormatUtils.ts": "8962fc5f7d02fd19fb2bce937692f9aae4fb15379a40d4ad4edd2634197e7abc", "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/types/Misc.ts": "205df5b124cb1e17cffccbc7cd920257af4744aea67308d18c05e7e4d12e9827", "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/types/PlatformShim.ts": "06f656f0d2bc20980ef77148455b662af10fe4b0e48d41566bf28e471eea4be1", - "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/types/StreamingInfoOptions.ts": "cd404a388a8d36bc28b60053851cd92495b666938552898c430c661ac4f7ad0b", "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/types/index.ts": "a28448751e5567b91cc60528a82999885630ed099c19318f265c4360537cff4e", "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/utils/Cache.ts": "fd90c88da32e9283adf065475a5cb4e680b5152a6bc06dd8a3dc9349358cab35", "https://raw.githubusercontent.com/LuanRT/YouTube.js/refs/tags/v13.1.0-deno/deno/src/utils/Constants.ts": "148e86c63bf222845ccf24508be7d93b599543cf14e73a78a870ffe92c834857", @@ -1036,12 +1096,13 @@ }, "workspace": { "dependencies": [ - "jsr:@hono/hono@^4.6.5", + "jsr:@hono/hono@^4.7.2", + "jsr:@luanrt/googlevideo@2", + "jsr:@luanrt/jintr@^3.2.1", "npm:@willsoto/node-konfig-core@5.0.0", "npm:@willsoto/node-konfig-file@3.0.0", "npm:@willsoto/node-konfig-toml-parser@3.0.0", - "npm:googlevideo@2.0.0", - "npm:jsdom@25.0.1", + "npm:jsdom@26.0.0", "npm:prom-client@^15.1.3" ] } diff --git a/src/lib/helpers/youtubePlayerHandling.ts b/src/lib/helpers/youtubePlayerHandling.ts index 55aefc9..8a082d9 100644 --- a/src/lib/helpers/youtubePlayerHandling.ts +++ b/src/lib/helpers/youtubePlayerHandling.ts @@ -2,8 +2,9 @@ import { ApiResponse, Innertube, YT } from "youtubei.js"; import { generateRandomString } from "youtubei.js/Utils"; import { compress, decompress } from "https://deno.land/x/brotli@0.1.7/mod.ts"; import { Store } from "@willsoto/node-konfig-core"; -import { failedRequests, successfulRequests, retryCount } from "metrics"; +import { failedRequests, retryCount, successfulRequests } from "metrics"; import { innertubeEmbeddedClient } from "../../main.ts"; +import type { BG } from "bgutils"; let youtubePlayerReqLocation = "youtubePlayerReq"; if (Deno.env.get("YT_PLAYER_REQ_LOCATION")) { if (Deno.env.has("DENO_COMPILED")) { @@ -30,23 +31,34 @@ if ((Deno.env.get("VIDEO_CACHE_ON_DISK")?.toLowerCase() ?? false) == "true") { kv = await Deno.openKv(); } -export const youtubePlayerParsing = async ( - innertubeClient: Innertube, - videoId: string, - konfigStore: Store, -): Promise => { - const cacheEnabled = konfigStore.get("cache.enabled"); +export const youtubePlayerParsing = async ({ + innertubeClient, + videoId, + konfigStore, + tokenMinter, + overrideCache = false, +}: { + innertubeClient: Innertube; + videoId: string; + konfigStore: Store; + tokenMinter: BG.WebPoMinter; + overrideCache?: boolean; +}): Promise => { + const cacheEnabled = overrideCache + ? false + : konfigStore.get("cache.enabled"); const videoCached = (await kv.get(["video_cache", videoId])) .value as Uint8Array; - if (videoCached != null && cacheEnabled == true) { + if (videoCached != null && cacheEnabled) { return JSON.parse(new TextDecoder().decode(decompress(videoCached))); } else { let youtubePlayerResponse = await youtubePlayerReq( innertubeClient, videoId, konfigStore, + tokenMinter, ); for (let retries = 1; retries <= (maxRetries as number); retries++) { @@ -57,8 +69,10 @@ export const youtubePlayerParsing = async ( ) { break; } - console.log(`[DEBUG] Got 'This helps protect our community', retrying request for ${videoId}. Retry ${retries} of ${maxRetries}`) - retryCount.inc() + console.log( + `[DEBUG] Got 'This helps protect our community', retrying request for ${videoId}. Retry ${retries} of ${maxRetries}`, + ); + retryCount.inc(); youtubePlayerResponse = await youtubePlayerReq( innertubeClient, videoId, @@ -67,10 +81,14 @@ export const youtubePlayerParsing = async ( } if ( - youtubePlayerResponse.data.playabilityStatus.status === "UNPLAYABLE" || - youtubePlayerResponse.data.playabilityStatus.status === "LOGIN_REQUIRED" || - youtubePlayerResponse.data.playabilityStatus.status === "CONTENT_CHECK_REQUIRED" || - youtubePlayerResponse.data.playabilityStatus.reason === "Sign in to confirm your age" + youtubePlayerResponse.data.playabilityStatus.status === + "UNPLAYABLE" || + youtubePlayerResponse.data.playabilityStatus.status === + "LOGIN_REQUIRED" || + youtubePlayerResponse.data.playabilityStatus.status === + "CONTENT_CHECK_REQUIRED" || + youtubePlayerResponse.data.playabilityStatus.reason === + "Sign in to confirm your age" ) { innertubeEmbeddedClient.session.context.client.visitorData = innertubeClient.session.context.client.visitorData; @@ -186,15 +204,6 @@ export const youtubePlayerParsing = async ( streamingData, videoDetails, microformat, - invidiousCompanion: { - "baseUrl": Deno.env.get("SERVER_BASE_URL") || - konfigStore.get("server.base_url") as string, - "external_videoplayback_proxy": - Deno.env.get("EXTERNAL_VIDEOPLAYBACK_PROXY") || - konfigStore.get( - "networking.external_videoplayback_proxy", - ) as string, - }, }))(videoData); if (videoData.playabilityStatus?.status == "OK") { @@ -215,9 +224,9 @@ export const youtubePlayerParsing = async ( }, ); })(); + } else { + failedRequests.inc(); } - } else { - failedRequests.inc(); } return videoOnlyNecessaryInfo; diff --git a/src/lib/helpers/youtubePlayerReq.ts b/src/lib/helpers/youtubePlayerReq.ts index 01d00ca..3f93104 100644 --- a/src/lib/helpers/youtubePlayerReq.ts +++ b/src/lib/helpers/youtubePlayerReq.ts @@ -1,11 +1,13 @@ import { ApiResponse, ClientType, Innertube } from "youtubei.js"; import { Store } from "@willsoto/node-konfig-core"; import NavigationEndpoint from "youtubei.js/NavigationEndpoint"; +import type { BG } from "bgutils"; export const youtubePlayerReq = async ( innertubeClient: Innertube, videoId: string, konfigStore: Store, + tokenMinter: BG.WebPoMinter, ): Promise => { const innertubeClientOauthEnabled = konfigStore.get( "youtube_session.oauth_enabled", @@ -23,7 +25,9 @@ export const youtubePlayerReq = async ( watchEndpoint: { videoId: videoId }, }); - return await watch_endpoint.call(innertubeClient.actions, { + const contentPoToken = await tokenMinter.mintAsWebsafeString(videoId); + + return watch_endpoint.call(innertubeClient.actions, { playbackContext: { contentPlaybackContext: { vis: 0, @@ -33,7 +37,7 @@ export const youtubePlayerReq = async ( }, }, serviceIntegrityDimensions: { - poToken: innertubeClient.session.po_token, + poToken: contentPoToken, }, client: innertubeClientUsed, }); diff --git a/src/lib/helpers/youtubeTranscriptsHandling.ts b/src/lib/helpers/youtubeTranscriptsHandling.ts new file mode 100644 index 0000000..93ede51 --- /dev/null +++ b/src/lib/helpers/youtubeTranscriptsHandling.ts @@ -0,0 +1,88 @@ +import { Innertube } from "youtubei.js"; +import type { CaptionTrackData } from "youtubei.js/PlayerCaptionsTracklist"; +import { HTTPException } from "hono/http-exception"; + +function createTemporalDuration(milliseconds: number) { + return new Temporal.Duration( + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + milliseconds, + ); +} + +const ESCAPE_SUBSTITUTIONS = { + "&": "&", + "<": "<", + ">": ">", + "\u200E": "‎", + "\u200F": "‏", + "\u00A0": " ", +}; + +export async function handleTranscripts( + innertubeClient: Innertube, + videoId: string, + selectedCaption: CaptionTrackData, +) { + const lines: string[] = ["WEBVTT"]; + + const info = await innertubeClient.getInfo(videoId); + const transcriptInfo = await (await info.getTranscript()).selectLanguage( + selectedCaption.name.text || "", + ); + const rawTranscriptLines = transcriptInfo.transcript.content?.body + ?.initial_segments; + + if (rawTranscriptLines == undefined) throw new HTTPException(404); + + rawTranscriptLines.forEach((line) => { + const timestampFormatOptions = { + style: "digital", + minutesDisplay: "always", + fractionalDigits: 3, + }; + + // Temporal.Duration.prototype.toLocaleString() is supposed to delegate to Intl.DurationFormat + // which Deno does not support. However, instead of following specs and having toLocaleString return + // the same toString() it seems to have its own implementation of Intl.DurationFormat, + // with its options parameter type incorrectly restricted to the same as the one for Intl.DateTimeFormatOptions + // even though they do not share the same arguments. + // + // The above matches the options parameter of Intl.DurationFormat, and the resulting output is as expected. + // Until this is fixed typechecking must be disabled for the two use cases below + // + // See + // https://docs.deno.com/api/web/~/Intl.DateTimeFormatOptions + // https://docs.deno.com/api/web/~/Temporal.Duration.prototype.toLocaleString + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Temporal/Duration/toLocaleString + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/DurationFormat/DurationFormat + + const start_ms = createTemporalDuration(Number(line.start_ms)).round({ + largestUnit: "year", + //@ts-ignore see above + }).toLocaleString("en-US", timestampFormatOptions); + + const end_ms = createTemporalDuration(Number(line.end_ms)).round({ + largestUnit: "year", + //@ts-ignore see above + }).toLocaleString("en-US", timestampFormatOptions); + const timestamp = `${start_ms} --> ${end_ms}`; + + const text = (line.snippet?.text || "").replace( + /[&<>‍‍\u200E\u200F\u00A0]/g, + (match: string) => + ESCAPE_SUBSTITUTIONS[ + match as keyof typeof ESCAPE_SUBSTITUTIONS + ], + ); + + lines.push(`${timestamp}\n${text}`); + }); + + return lines.join("\n\n"); +} diff --git a/src/lib/jobs/potoken.ts b/src/lib/jobs/potoken.ts index d7dce03..c4d4c5f 100644 --- a/src/lib/jobs/potoken.ts +++ b/src/lib/jobs/potoken.ts @@ -1,9 +1,13 @@ -import { BG } from "bgutils"; -import type { BgConfig } from "bgutils"; +import { BG, buildURL, GOOG_API_KEY, USER_AGENT } from "bgutils"; +import type { WebPoSignalOutput } from "bgutils"; import { JSDOM } from "jsdom"; import { Innertube, UniversalCache } from "youtubei.js"; import { Store } from "@willsoto/node-konfig-core"; -import { externalTokenGeneratorFail, poTokenFail } from "metrics"; +import { poTokenFail } from "metrics"; +import { + youtubePlayerParsing, + youtubeVideoInfo, +} from "../helpers/youtubePlayerHandling.ts"; let getFetchClientLocation = "getFetchClient"; if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) { if (Deno.env.has("DENO_COMPILED")) { @@ -22,96 +26,58 @@ export const poTokenGenerate = async ( innertubeClient: Innertube, konfigStore: Store>, innertubeClientCache: UniversalCache, -): Promise => { - const externalTokenGenerator = konfigStore.get( - "server.external_token_generator", - ) as string; - const externalTokenGeneratorKey = konfigStore.get( - "server.external_token_generator_key", - ) as string; - const requestKey = "O43z0dpjhgX20SCx4KAo"; - - if (externalTokenGenerator != "" && externalTokenGenerator != undefined) { - console.log( - "poTokenGenerate: using external token generator at " + - externalTokenGenerator, - ); - try { - let response: Response; - if ( - externalTokenGeneratorKey != "" && - externalTokenGeneratorKey != undefined - ) { - response = await fetch( - `${externalTokenGenerator}/generate`, - { - headers: { - "Authorization": - `Bearer ${externalTokenGeneratorKey}`, - }, - }, - ); - if (response.status == 401) { - throw new Error( - `Key '${externalTokenGeneratorKey}' is invalid!`, - ); - } - } else { - response = await fetch( - `${externalTokenGenerator}/generate`, - ); - } - const data = await response.json(); - return (await Innertube.create({ - po_token: data.potoken, - visitor_data: data.visitorData, - fetch: getFetchClient(konfigStore), - cache: innertubeClientCache, - generate_session_locally: true, - })); - } catch (e) { - console.error( - "poTokenGenerate: error fetch token from the external token generator: " + - e, - ); - console.log("poTokenGenerate: Using built-in token generator"); - externalTokenGeneratorFail.inc(); - } - } - +): Promise<{ innertubeClient: Innertube; tokenMinter: BG.WebPoMinter }> => { if (innertubeClient.session.po_token) { - innertubeClient = await Innertube.create({ retrieve_player: false }); + innertubeClient = await Innertube.create({ + enable_session_cache: false, + user_agent: USER_AGENT, + retrieve_player: false, + }); } + const fetchImpl = await getFetchClient(konfigStore); + const visitorData = innertubeClient.session.context.client.visitorData; if (!visitorData) { throw new Error("Could not get visitor data"); } - const dom = new JSDOM(); + const dom = new JSDOM( + '', + { + url: "https://www.youtube.com/", + referrer: "https://www.youtube.com/", + userAgent: USER_AGENT, + }, + ); Object.assign(globalThis, { window: dom.window, document: dom.window.document, + location: dom.window.location, + origin: dom.window.origin, }); - const bgConfig: BgConfig = { - fetch: getFetchClient(konfigStore), - globalObj: globalThis, - identifier: visitorData, - requestKey, - }; + if (!Reflect.has(globalThis, "navigator")) { + Object.defineProperty(globalThis, "navigator", { + value: dom.window.navigator, + }); + } - const bgChallenge = await BG.Challenge.create(bgConfig); - - if (!bgChallenge) { - poTokenFail.inc(); + const challengeResponse = await innertubeClient.getAttestationChallenge( + "ENGAGEMENT_TYPE_UNBOUND", + ); + if (!challengeResponse.bg_challenge) { throw new Error("Could not get challenge"); } - const interpreterJavascript = bgChallenge.interpreterJavascript - .privateDoNotAccessOrElseSafeScriptWrappedValue; + const interpreterUrl = challengeResponse.bg_challenge.interpreter_url + .private_do_not_access_or_else_trusted_resource_url_wrapped_value; + const bgScriptResponse = await fetchImpl( + `http:${interpreterUrl}`, + ); + const interpreterJavascript = await bgScriptResponse.text(); if (interpreterJavascript) { new Function(interpreterJavascript)(); @@ -120,19 +86,103 @@ export const poTokenGenerate = async ( throw new Error("Could not load VM"); } - const poTokenResult = await BG.PoToken.generate({ - program: bgChallenge.program, - globalName: bgChallenge.globalName, - bgConfig, + // Botguard currently surfaces a "Not implemented" error here, due to the environment + // not having a valid Canvas API in JSDOM. At the time of writing, this doesn't cause + // any issues as the Canvas check doesn't appear to be an enforced element of the checks + console.log( + '[INFO] the "Not implemented: HTMLCanvasElement.prototype.getContext" error is normal. Please do not open a bug report about it.', + ); + const botguard = await BG.BotGuardClient.create({ + program: challengeResponse.bg_challenge.program, + globalName: challengeResponse.bg_challenge.global_name, + globalObj: globalThis, }); - await BG.PoToken.generatePlaceholder(visitorData); + const webPoSignalOutput: WebPoSignalOutput = []; + const botguardResponse = await botguard.snapshot({ webPoSignalOutput }); + const requestKey = "O43z0dpjhgX20SCx4KAo"; - return (await Innertube.create({ - po_token: poTokenResult.poToken, + const integrityTokenResponse = await fetchImpl( + buildURL("GenerateIT", true), + { + method: "POST", + headers: { + "content-type": "application/json+protobuf", + "x-goog-api-key": GOOG_API_KEY, + "x-user-agent": "grpc-web-javascript/0.1", + "user-agent": USER_AGENT, + }, + body: JSON.stringify([requestKey, botguardResponse]), + }, + ); + + const response = await integrityTokenResponse.json() as unknown[]; + + if (typeof response[0] !== "string") { + throw new Error("Could not get integrity token"); + } + + const integrityTokenBasedMinter = await BG.WebPoMinter.create({ + integrityToken: response[0], + }, webPoSignalOutput); + + const sessionPoToken = await integrityTokenBasedMinter.mintAsWebsafeString( + visitorData, + ); + + const instantiatedInnertubeClient = await Innertube.create({ + enable_session_cache: false, + po_token: sessionPoToken, visitor_data: visitorData, fetch: getFetchClient(konfigStore), cache: innertubeClientCache, generate_session_locally: true, - })); + }); + + try { + const feed = await instantiatedInnertubeClient.getTrending(); + // get all videos and shuffle them randomly to avoid using the same trending video over and over + const videos = feed.videos + .filter((video) => video.type === "Video") + .map((value) => ({ value, sort: Math.random() })) + .sort((a, b) => a.sort - b.sort) + .map(({ value }) => value); + + const video = videos.find((video) => "id" in video); + if (!video) { + throw new Error("no videos with id found in trending"); + } + + const youtubePlayerResponseJson = await youtubePlayerParsing({ + innertubeClient: instantiatedInnertubeClient, + videoId: video.id, + konfigStore, + tokenMinter: integrityTokenBasedMinter, + overrideCache: true, + }); + const videoInfo = youtubeVideoInfo( + instantiatedInnertubeClient, + youtubePlayerResponseJson, + ); + const validFormat = videoInfo.streaming_data?.adaptive_formats[0]; + if (!validFormat) { + throw new Error( + "failed to find valid video with adaptive format to check token against", + ); + } + const result = await fetchImpl(validFormat?.url, { method: "HEAD" }); + if (result.status !== 200) { + throw new Error( + `did not get a 200 when checking video, got ${result.status} instead`, + ); + } + } catch (err) { + console.log("Failed to get valid PO token, will retry", { err }); + throw err; + } + + return { + innertubeClient: instantiatedInnertubeClient, + tokenMinter: integrityTokenBasedMinter, + }; }; diff --git a/src/lib/types/HonoVariables.ts b/src/lib/types/HonoVariables.ts index 2dea7be..c2f0c78 100644 --- a/src/lib/types/HonoVariables.ts +++ b/src/lib/types/HonoVariables.ts @@ -1,7 +1,9 @@ import { Innertube } from "youtubei.js"; +import { BG } from "bgutils"; import type { konfigLoader } from "../helpers/konfigLoader.ts"; export type HonoVariables = { innertubeClient: Innertube; konfigStore: Awaited>; + tokenMinter: BG.WebPoMinter; }; diff --git a/src/main.ts b/src/main.ts index 930d7ef..d1b9ee8 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,8 +2,12 @@ import { Hono } from "hono"; import { routes } from "./routes/index.ts"; import { ClientType, Innertube, UniversalCache } from "youtubei.js"; import { poTokenGenerate } from "./lib/jobs/potoken.ts"; +import { USER_AGENT } from "bgutils"; import { konfigLoader } from "./lib/helpers/konfigLoader.ts"; +import { retry } from "jsr:@std/async"; import type { HonoVariables } from "./lib/types/HonoVariables.ts"; +import type { BG } from "bgutils"; + let getFetchClientLocation = "getFetchClient"; if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) { if (Deno.env.has("DENO_COMPILED")) { @@ -15,32 +19,15 @@ if (Deno.env.get("GET_FETCH_CLIENT_LOCATION")) { ) as string; } } - -const args = Deno.args; -const konfigStore = await konfigLoader(); -const host = Deno.env.get("HOST") || konfigStore.get("server.host") as string; -const port = Deno.env.get("PORT") || konfigStore.get("server.port") as string; - -if (args?.[0] == "healthcheck") { - try { - const response = await fetch(`http://${host}:${port}/healthz`); - if (response.status === 200) { - Deno.exit(0); - } else { - Deno.exit(1); - } - } catch (_) { - Deno.exit(1); - } -} - const { getFetchClient } = await import(getFetchClientLocation); declare module "hono" { interface ContextVariableMap extends HonoVariables {} } const app = new Hono(); +const konfigStore = await konfigLoader(); +let tokenMinter: BG.WebPoMinter; let innertubeClient: Innertube; let innertubeClientFetchPlayer = true; const innertubeClientOauthEnabled = konfigStore.get( @@ -73,10 +60,12 @@ if (!innertubeClientOauthEnabled) { } innertubeClient = await Innertube.create({ + enable_session_cache: false, cache: innertubeClientCache, retrieve_player: innertubeClientFetchPlayer, fetch: getFetchClient(konfigStore), cookie: innertubeClientCookies || undefined, + user_agent: USER_AGENT, }); export const innertubeEmbeddedClient = await Innertube.create({ @@ -93,24 +82,31 @@ console.log("[INFO] po_token refresh interval set to", poTokenRefreshInterval); if (!innertubeClientOauthEnabled) { if (innertubeClientJobPoTokenEnabled) { - innertubeClient = await poTokenGenerate( - innertubeClient, - konfigStore, - innertubeClientCache as UniversalCache, - ); + ({ innertubeClient, tokenMinter } = await retry( + poTokenGenerate.bind( + poTokenGenerate, + innertubeClient, + konfigStore, + innertubeClientCache as UniversalCache, + ), + { minTimeout: 1_000, maxTimeout: 60_000, multiplier: 5, jitter: 0 }, + )); } setInterval( async () => { if (innertubeClientJobPoTokenEnabled) { - innertubeClient = await poTokenGenerate( + ({ innertubeClient, tokenMinter } = await poTokenGenerate( innertubeClient, konfigStore, innertubeClientCache, - ); + )); } else { innertubeClient = await Innertube.create({ + enable_session_cache: false, cache: innertubeClientCache, + fetch: getFetchClient(konfigStore), retrieve_player: innertubeClientFetchPlayer, + user_agent: USER_AGENT, }); } }, @@ -140,6 +136,7 @@ if (!innertubeClientOauthEnabled) { app.use("*", async (c, next) => { c.set("innertubeClient", innertubeClient); + c.set("tokenMinter", tokenMinter); c.set("konfigStore", konfigStore); await next(); }); @@ -147,6 +144,7 @@ app.use("*", async (c, next) => { routes(app, konfigStore); Deno.serve({ - port: Number(port), - hostname: host + port: Number(Deno.env.get("PORT")) || + konfigStore.get("server.port") as number, + hostname: Deno.env.get("HOST") || konfigStore.get("server.host") as string, }, app.fetch); diff --git a/src/routes/index.ts b/src/routes/index.ts index 8b815f6..a0c3178 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -6,6 +6,7 @@ import { bearerAuth } from "hono/bearer-auth"; import youtubeApiPlayer from "./youtube_api_routes/player.ts"; import invidiousRouteLatestVersion from "./invidious_routes/latestVersion.ts"; import invidiousRouteDashManifest from "./invidious_routes/dashManifest.ts"; +import invidiousCaptionsApi from "./invidious_routes/captions.ts"; import videoPlaybackProxy from "./videoPlaybackProxy.ts"; import metrics from "metrics"; import health from "./health.ts"; @@ -37,6 +38,8 @@ export const routes = ( app.route("/videoplayback", videoPlaybackProxy); } app.route("/metrics", metrics); + app.route("/api/v1/captions", invidiousCaptionsApi); + app.route("/videoplayback", videoPlaybackProxy); app.route("/healthz", health); app.route("/info", info); diff --git a/src/routes/invidious_routes/captions.ts b/src/routes/invidious_routes/captions.ts new file mode 100644 index 0000000..e73a85a --- /dev/null +++ b/src/routes/invidious_routes/captions.ts @@ -0,0 +1,95 @@ +import { Hono } from "hono"; +import type { HonoVariables } from "../../lib/types/HonoVariables.ts"; +import { verifyRequest } from "../../lib/helpers/verifyRequest.ts"; +import { + youtubePlayerParsing, + youtubeVideoInfo, +} from "../../lib/helpers/youtubePlayerHandling.ts"; +import type { CaptionTrackData } from "youtubei.js/PlayerCaptionsTracklist"; +import { handleTranscripts } from "../../lib/helpers/youtubeTranscriptsHandling.ts"; +import { HTTPException } from "hono/http-exception"; + +interface AvailableCaption { + label: string; + languageCode: string; + url: string; +} + +const captionsHandler = new Hono<{ Variables: HonoVariables }>(); +captionsHandler.get("/:videoId", async (c) => { + const { videoId } = c.req.param(); + const konfigStore = c.get("konfigStore"); + + const check = c.req.query("check"); + + if (konfigStore.get("server.verify_requests") && check == undefined) { + throw new HTTPException(400, { + res: new Response("No check ID."), + }); + } else if (konfigStore.get("server.verify_requests") && check) { + if (verifyRequest(check, videoId, konfigStore) === false) { + throw new HTTPException(400, { + res: new Response("ID incorrect."), + }); + } + } + + const innertubeClient = c.get("innertubeClient"); + + const youtubePlayerResponseJson = await youtubePlayerParsing({ + innertubeClient, + videoId, + konfigStore, + tokenMinter: c.get("tokenMinter"), + }); + + const videoInfo = youtubeVideoInfo( + innertubeClient, + youtubePlayerResponseJson, + ); + + const captionsTrackArray = videoInfo.captions?.caption_tracks; + if (captionsTrackArray == undefined) throw new HTTPException(404); + + const label = c.req.query("label"); + const lang = c.req.query("lang"); + + // Show all available captions when a specific one is not selected + if (label == undefined && lang == undefined) { + const invidiousAvailableCaptionsArr: AvailableCaption[] = []; + + for (const caption_track of captionsTrackArray) { + invidiousAvailableCaptionsArr.push({ + label: caption_track.name.text || "", + languageCode: caption_track.language_code, + url: `/api/v1/captions/${videoId}?label=${ + encodeURIComponent(caption_track.name.text || "") + }`, + }); + } + + return c.json({ captions: invidiousAvailableCaptionsArr }); + } + + // Extract selected caption + let filterSelected: CaptionTrackData[]; + + if (lang) { + filterSelected = captionsTrackArray.filter((c: CaptionTrackData) => + c.language_code === lang + ); + } else { + filterSelected = captionsTrackArray.filter((c: CaptionTrackData) => + c.name.text === label + ); + } + + if (filterSelected.length == 0) throw new HTTPException(404); + + c.header("Content-Type", "text/vtt; charset=UTF-8"); + return c.body( + await handleTranscripts(innertubeClient, videoId, filterSelected[0]), + ); +}); + +export default captionsHandler; diff --git a/src/routes/invidious_routes/dashManifest.ts b/src/routes/invidious_routes/dashManifest.ts index c8ff8b8..d036d47 100644 --- a/src/routes/invidious_routes/dashManifest.ts +++ b/src/routes/invidious_routes/dashManifest.ts @@ -36,11 +36,12 @@ dashManifest.get("/:videoId", async (c) => { } } - const youtubePlayerResponseJson = await youtubePlayerParsing( + const youtubePlayerResponseJson = await youtubePlayerParsing({ innertubeClient, videoId, konfigStore, - ); + tokenMinter: c.get("tokenMinter"), + }); const videoInfo = youtubeVideoInfo( innertubeClient, youtubePlayerResponseJson, @@ -132,7 +133,7 @@ dashManifest.get("/:videoId", async (c) => { captions, undefined, ); - return c.body(dashFile.replaceAll("&", "&")); + return c.body(dashFile); } }); diff --git a/src/routes/invidious_routes/latestVersion.ts b/src/routes/invidious_routes/latestVersion.ts index b3bb41f..bb688c4 100644 --- a/src/routes/invidious_routes/latestVersion.ts +++ b/src/routes/invidious_routes/latestVersion.ts @@ -20,7 +20,7 @@ latestVersion.get("/", async (c) => { } const innertubeClient = c.get("innertubeClient"); - const konfigStore = await c.get("konfigStore"); + const konfigStore = c.get("konfigStore"); if (konfigStore.get("server.verify_requests") && check == undefined) { throw new HTTPException(400, { @@ -34,11 +34,12 @@ latestVersion.get("/", async (c) => { } } - const youtubePlayerResponseJson = await youtubePlayerParsing( + const youtubePlayerResponseJson = await youtubePlayerParsing({ innertubeClient, - id, + videoId: id, konfigStore, - ); + tokenMinter: c.get("tokenMinter"), + }); const videoInfo = youtubeVideoInfo( innertubeClient, youtubePlayerResponseJson, diff --git a/src/routes/videoPlaybackProxy.ts b/src/routes/videoPlaybackProxy.ts index 1f17b15..43bada8 100644 --- a/src/routes/videoPlaybackProxy.ts +++ b/src/routes/videoPlaybackProxy.ts @@ -33,8 +33,6 @@ videoPlaybackProxy.get("/", async (c) => { ({ host, c: client, expire } = c.req.query()); } - const rangeHeader = c.req.header("range") as string | undefined; - if (host == undefined || !/[\w-]+.googlevideo.com/.test(host)) { throw new HTTPException(400, { res: new Response("Host query string do not match or undefined."), @@ -59,10 +57,12 @@ videoPlaybackProxy.get("/", async (c) => { } queryParams.delete("host"); - if (rangeHeader) { + const rangeHeader = c.req.header("range"); + const requestBytes = rangeHeader ? rangeHeader.split("=")[1] : null; + if (requestBytes) { queryParams.append( "range", - rangeHeader.split("=")[1], + requestBytes, ); } @@ -109,7 +109,7 @@ videoPlaybackProxy.get("/", async (c) => { ); } - const headersForResponse = { + const headersForResponse: Record = { "content-length": googlevideoResponse.headers.get("content-length") || "", "access-control-allow-origin": "*", @@ -119,8 +119,14 @@ videoPlaybackProxy.get("/", async (c) => { "last-modified": googlevideoResponse.headers.get("last-modified") || "", }; + let responseStatus = googlevideoResponse.status; + if (requestBytes && responseStatus == 200) { + responseStatus = 206; + headersForResponse["content-range"] = `bytes ${requestBytes}/*`; + } + return new Response(googlevideoResponse.body, { - status: googlevideoResponse.status, + status: responseStatus, statusText: googlevideoResponse.statusText, headers: headersForResponse, }); diff --git a/src/routes/youtube_api_routes/player.ts b/src/routes/youtube_api_routes/player.ts index 30b4206..4fc8848 100644 --- a/src/routes/youtube_api_routes/player.ts +++ b/src/routes/youtube_api_routes/player.ts @@ -47,17 +47,14 @@ player.post("/player", async (c) => { const innertubeClient = c.get("innertubeClient"); const konfigStore = c.get("konfigStore"); if (jsonReq.videoId) { - const yt = await youtubePlayerParsing( - innertubeClient, - jsonReq.videoId, - konfigStore, + return c.json( + await youtubePlayerParsing({ + innertubeClient, + videoId: jsonReq.videoId, + konfigStore, + tokenMinter: c.get("tokenMinter"), + }), ); - errors.forEach((error) => { - if (error.check(yt)) { - error.action(); - } - }); - return c.json(yt); } });