Compare commits

...

6 commits

Author SHA1 Message Date
7f16feaff6
partial support for history details in history page 2024-12-24 14:21:42 -03:00
ee6d2b5620
Channels: Fix "Youtube API returned error 400"
All checks were successful
Invidious CI / build (push) Successful in 6m10s
commit 301aeffa78
Author: Samantaz Fox <coding@samantaz.fr>
Date:   Fri Nov 8 13:54:05 2024 +0100

    Channels: Multiple small fixes

    Fix the "newest" link not being bold when 'sort_by' uses the default value
    Show 60 videos per page, rather than 30

commit d27a5e7fae
Author: Samantaz Fox <coding@samantaz.fr>
Date:   Fri Nov 8 13:33:46 2024 +0100

    Channels: Rename ctoken generator functions as requested

commit afc5b27d83
Author: Samantaz Fox <coding@samantaz.fr>
Date:   Fri Nov 8 13:32:44 2024 +0100

    Extractors: Add support for shortsLockupViewModel

    The 'shortsLockupViewModel' structure is used in the channel "shorts" tab

commit 1a5047aad9
Author: Samantaz Fox <coding@samantaz.fr>
Date:   Fri Nov 8 12:33:14 2024 +0100

    Extractors: Add support for lockupViewModel

    The 'lockupViewModel' structure is used in the channel "podcasts" tab

commit 82248fad02
Author: Samantaz Fox <coding@samantaz.fr>
Date:   Thu Nov 7 23:00:18 2024 +0100

    Channels: Add sort options to shorts

commit cbc546f032
Author: Samantaz Fox <coding@samantaz.fr>
Date:   Thu Nov 7 22:54:21 2024 +0100

    Channels: Add function to generate the new ctoken objects

commit c243d08afb
Author: Brahim Hadriche <brahim.hadriche@gmail.com>
Date:   Wed Oct 30 13:38:13 2024 -0400

    refactor

commit ee72809282
Author: Brahim Hadriche <brahim.hadriche@gmail.com>
Date:   Sat Oct 26 12:40:31 2024 -0400

    [Alternative] Fix for channel live videos
2024-11-08 11:30:51 -03:00
4da3936b5a
Remove old code that is done on the Openresty side
All checks were successful
Invidious CI / build (push) Successful in 5m26s
2024-11-07 21:12:59 -03:00
b95a8bfbd3
Update CI
All checks were successful
Invidious CI / build (push) Successful in 5m10s
2024-11-05 00:36:24 -03:00
4849286814
Videos: Add support for OpenGraph videos
All checks were successful
Invidious CI / build (push) Successful in 5m18s
To support OpenGraph clients like Discord and other platforms able to
pull the video from the OpenGraph metadata.
2024-11-04 23:49:45 -03:00
Brahim Hadriche
e3da8f408d
[Alternative] Fix for channel live videos
All checks were successful
Invidious CI / build (push) Successful in 6m30s
Signed-off-by: Fijxu <fijxu@nadeko.net>

refactor

Signed-off-by: Fijxu <fijxu@nadeko.net>
2024-11-01 13:56:32 -03:00
14 changed files with 278 additions and 145 deletions

View file

@ -37,13 +37,15 @@ jobs:
type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
- uses: https://code.forgejo.org/docker/build-push-action@v5
- uses: https://code.forgejo.org/docker/build-push-action@v6
name: Build images
with:
context: .
file: docker/Dockerfile
tags: ${{ steps.meta.outputs.tags }}
platforms: linux/amd64
# cache-from: type=gha
# cache-to: type=gha,mode=max
push: true
build-args: |
"release=1"

View file

@ -1,4 +1,4 @@
FROM crystallang/crystal:1.12.1-alpine AS builder
FROM crystallang/crystal:1.14.0-alpine AS builder
RUN apk add --no-cache sqlite-static yaml-static

View file

@ -1,78 +1,3 @@
def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
object_inner_2 = {
"2:0:embedded" => {
"1:0:varint" => 0_i64,
},
"5:varint" => 50_i64,
"6:varint" => 1_i64,
"7:varint" => (page * 30).to_i64,
"9:varint" => 1_i64,
"10:varint" => 0_i64,
}
object_inner_2_encoded = object_inner_2
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
content_type_numerical =
case content_type
when "videos" then 15
when "livestreams" then 14
else 15 # Fallback to "videos"
end
sort_by_numerical =
case sort_by
when "newest" then 1_i64
when "popular" then 2_i64
when "oldest" then 4_i64
else 1_i64 # Fallback to "newest"
end
object_inner_1 = {
"110:embedded" => {
"3:embedded" => {
"#{content_type_numerical}:embedded" => {
"1:embedded" => {
"1:string" => object_inner_2_encoded,
},
"2:embedded" => {
"1:string" => "00000000-0000-0000-0000-000000000000",
},
"3:varint" => sort_by_numerical,
},
},
},
}
object_inner_1_encoded = object_inner_1
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:string" => object_inner_1_encoded,
"35:string" => "browse-feed#{ucid}videos102",
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
def make_initial_content_ctoken(ucid, content_type, sort_by) : String
return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by)
end
module Invidious::Channel::Tabs
extend self
@ -101,7 +26,7 @@ module Invidious::Channel::Tabs
end
def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by)
continuation ||= make_initial_videos_ctoken(ucid, sort_by)
initial_data = YoutubeAPI.browse(continuation: continuation)
return extract_items(initial_data, author, ucid)
@ -130,14 +55,10 @@ module Invidious::Channel::Tabs
# Shorts
# -------------------
def get_shorts(channel : AboutChannel, continuation : String? = nil)
if continuation.nil?
# EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
# TODO: try to extract the continuation tokens that allows other sorting options
initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
else
initial_data = YoutubeAPI.browse(continuation: continuation)
end
def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
continuation ||= make_initial_shorts_ctoken(channel.ucid, sort_by)
initial_data = YoutubeAPI.browse(continuation: continuation)
return extract_items(initial_data, channel.author, channel.ucid)
end
@ -145,9 +66,8 @@ module Invidious::Channel::Tabs
# Livestreams
# -------------------
def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest")
continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by)
def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
continuation ||= make_initial_livestreams_ctoken(channel.ucid, sort_by)
initial_data = YoutubeAPI.browse(continuation: continuation)
return extract_items(initial_data, channel.author, channel.ucid)
@ -171,4 +91,100 @@ module Invidious::Channel::Tabs
return items, next_continuation
end
# -------------------
# C-tokens
# -------------------
private def sort_options_videos_short(sort_by : String)
case sort_by
when "newest" then return 4_i64
when "popular" then return 2_i64
when "oldest" then return 5_i64
else return 4_i64 # Fallback to "newest"
end
end
# Generate the initial "continuation token" to get the first page of the
# "videos" tab. The following page requires the ctoken provided in that
# first page, and so on.
private def make_initial_videos_ctoken(ucid : String, sort_by = "newest")
object = {
"15:embedded" => {
"2:string" => "\n$00000000-0000-0000-0000-000000000000",
"4:varint" => sort_options_videos_short(sort_by),
},
}
return channel_ctoken_wrap(ucid, object)
end
# Generate the initial "continuation token" to get the first page of the
# "shorts" tab. The following page requires the ctoken provided in that
# first page, and so on.
private def make_initial_shorts_ctoken(ucid : String, sort_by = "newest")
object = {
"10:embedded" => {
"2:embedded" => {
"1:string" => "00000000-0000-0000-0000-000000000000",
},
"4:varint" => sort_options_videos_short(sort_by),
},
}
return channel_ctoken_wrap(ucid, object)
end
# Generate the initial "continuation token" to get the first page of the
# "livestreams" tab. The following page requires the ctoken provided in that
# first page, and so on.
private def make_initial_livestreams_ctoken(ucid : String, sort_by = "newest")
sort_by_numerical =
case sort_by
when "newest" then 12_i64
when "popular" then 14_i64
when "oldest" then 13_i64
else 12_i64 # Fallback to "newest"
end
object = {
"14:embedded" => {
"2:embedded" => {
"1:string" => "00000000-0000-0000-0000-000000000000",
},
"5:varint" => sort_by_numerical,
},
}
return channel_ctoken_wrap(ucid, object)
end
# The protobuf structure common between videos/shorts/livestreams
private def channel_ctoken_wrap(ucid : String, object)
object_inner = {
"110:embedded" => {
"3:embedded" => object,
},
}
object_inner_encoded = object_inner
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:string" => object_inner_encoded,
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
end

View file

@ -204,6 +204,8 @@ class Config
property ignore_user_tokens : Bool = false
property server_id_cookie_name : String = "INVIDIOUS_SERVER_ID"
{% if flag?(:linux) %}
property reload_config_automatically : Bool = true
{% end %}

View file

@ -49,14 +49,14 @@ module Invidious::Database::Users
PG_DB.exec(request, user.watched, user.email)
end
def mark_watched(user : User, vid : String)
def mark_watched(user : User, history_details : JSON::Any)
request = <<-SQL
UPDATE users
SET watched = array_append(array_remove(watched, $1), $1)
SET watched = array_append(array_remove(watched, $1::jsonb), $1::jsonb)
WHERE email = $2
SQL
PG_DB.exec(request, vid, user.email)
PG_DB.exec(request, history_details, user.email)
end
def mark_unwatched(user : User, vid : String)

View file

@ -86,7 +86,7 @@ module Invidious::Routes::API::V1::Authenticated
return error_json(400, "Invalid video id.")
end
Invidious::Database::Users.mark_watched(user, id)
# Invidious::Database::Users.mark_watched(user, id)
env.response.status_code = 204
end

View file

@ -20,10 +20,11 @@ module Invidious::Routes::Channels
sort_by = env.params.query["sort_by"]?.try &.downcase
if channel.auto_generated
sort_by ||= "last"
sort_options = {"last", "oldest", "newest"}
items, next_continuation = fetch_channel_playlists(
channel.ucid, channel.author, continuation, (sort_by || "last")
channel.ucid, channel.author, continuation, sort_by
)
items.uniq! do |item|
@ -49,9 +50,11 @@ module Invidious::Routes::Channels
end
next_continuation = nil
else
sort_by ||= "newest"
sort_options = {"newest", "oldest", "popular"}
items, next_continuation = Channel::Tabs.get_videos(
channel, continuation: continuation, sort_by: (sort_by || "newest")
items, next_continuation = Channel::Tabs.get_60_videos(
channel, continuation: continuation, sort_by: sort_by
)
end
end
@ -82,13 +85,12 @@ module Invidious::Routes::Channels
end
next_continuation = nil
else
# TODO: support sort option for shorts
sort_by = ""
sort_options = [] of String
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
sort_options = {"newest", "oldest", "popular"}
# Fetch items and continuation token
items, next_continuation = Channel::Tabs.get_shorts(
channel, continuation: continuation
channel, continuation: continuation, sort_by: sort_by
)
end

View file

@ -1,18 +0,0 @@
{% skip_file if flag?(:api_only) %}
module Invidious::Routes::BackendSwitcher
def self.switch(env)
referer = get_referer(env)
backend_id = env.params.query["backend_id"]
# Checks if there is any alternative domain, like a second domain name,
# TOR or I2P address
if alt = CONFIG.alternative_domains.index(env.request.headers["Host"])
env.response.cookies["SERVER_ID"] = Invidious::User::Cookies.server_id(CONFIG.alternative_domains[alt], backend_id)
else
env.response.cookies["SERVER_ID"] = Invidious::User::Cookies.server_id(CONFIG.domain, backend_id)
end
env.redirect referer
end
end

View file

@ -76,8 +76,14 @@ module Invidious::Routes::Watch
end
env.params.query.delete_all("iv_load_policy")
history_details = JSON::Any.new({
"id" => JSON::Any.new(id),
"title" => JSON::Any.new(video.title),
"author" => JSON::Any.new(video.author)
})
if watched && preferences.watch_history
Invidious::Database::Users.mark_watched(user.as(User), id)
Invidious::Database::Users.mark_watched(user.as(User), history_details)
end
if CONFIG.enable_user_notifications && notifications && notifications.includes? id
@ -152,6 +158,7 @@ module Invidious::Routes::Watch
end
end
# Removes non default audio tracks
audio_streams.reject! do |z|
z if z.dig?("audioTrack", "audioIsDefault") == false
end
@ -218,6 +225,12 @@ module Invidious::Routes::Watch
captions: video.captions
)
begin
video_url = fmt_stream[0]["url"].to_s
rescue
video_url = nil
end
templated "watch"
end
@ -279,9 +292,9 @@ module Invidious::Routes::Watch
case action
when "action_mark_watched"
Invidious::Database::Users.mark_watched(user, id)
# Invidious::Database::Users.mark_watched(user, history_details)
when "action_mark_unwatched"
Invidious::Database::Users.mark_unwatched(user, id)
# Invidious::Database::Users.mark_unwatched(user, id)
else
return error_json(400, "Unsupported action #{action}")
end

View file

@ -21,7 +21,6 @@ module Invidious::Routing
get "/privacy", Routes::Misc, :privacy
get "/licenses", Routes::Misc, :licenses
get "/redirect", Routes::Misc, :cross_instance_redirect
get "/switchbackend", Routes::BackendSwitcher, :switch
self.register_channel_routes
self.register_watch_routes

View file

@ -45,18 +45,5 @@ struct Invidious::User
samesite: HTTP::Cookie::SameSite::Lax
)
end
# Server ID (SERVER_ID) cookie used for Sticky Sessions
# Parameter "domain" comes from the global config
def server_id(domain : String?, server_id) : HTTP::Cookie
return HTTP::Cookie.new(
name: "SERVER_ID",
domain: domain,
value: server_id,
secure: false,
http_only: true,
samesite: HTTP::Cookie::SameSite::Lax
)
end
end
end

View file

@ -1,7 +1,7 @@
<%
locale = env.get("preferences").as(Preferences).locale
dark_mode = env.get("preferences").as(Preferences).dark_mode
current_backend = env.request.cookies["SERVER_ID"]?.try &.value
current_backend = env.request.cookies[CONFIG.server_id_cookie_name]?.try &.value
%>
<!DOCTYPE html>
<html lang="<%= locale %>">
@ -310,7 +310,7 @@
</div>
<hr/>
<div class="footer-footer">
<div class="box">You are currently using Backend: <%= current_backend %></p>
<div class="box">You are currently using Backend: <%= current_backend %></div>
<span class="left">
<% if CONFIG.modified_source_code_url %>
<%= translate(locale, "footer_current_version_modified") %>

View file

@ -13,11 +13,13 @@
<meta property="og:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg">
<meta property="og:description" content="<%= HTML.escape(video.short_description) %>">
<meta property="og:type" content="video.other">
<meta property="og:video:url" content="<%= HOST_URL %>/embed/<%= video.id %>">
<meta property="og:video:secure_url" content="<%= HOST_URL %>/embed/<%= video.id %>">
<meta property="og:video:type" content="text/html">
<meta property="og:video:width" content="1280">
<meta property="og:video:height" content="720">
<!-- This shouldn't be empty, ever. -->
<meta property="og:video" content="<%= video_url %>">
<meta property="og:video:url" content="<%= video_url %>">
<meta property="og:video:secure_url" content="<%= video_url %>">
<meta property="og:video:type" content="video/mp4">
<meta property="og:video:width" content="640">
<meta property="og:video:height" content="360">
<meta name="twitter:card" content="player">
<meta name="twitter:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>">
<meta name="twitter:title" content="<%= title %>">

View file

@ -467,9 +467,9 @@ private module Parsers
# Parses an InnerTube richItemRenderer into a SearchVideo.
# Returns nil when the given object isn't a RichItemRenderer
#
# A richItemRenderer seems to be a simple wrapper for a videoRenderer, used
# by the result page for hashtags and for the podcast tab on channels.
# It is located inside a continuationItems container for hashtags.
# A richItemRenderer seems to be a simple wrapper for a various other types,
# used on the hashtags result page and the channel podcast tab. It is located
# itself inside a richGridRenderer container.
#
module RichItemRendererParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
@ -482,6 +482,8 @@ private module Parsers
child = VideoRendererParser.process(item_contents, author_fallback)
child ||= ReelItemRendererParser.process(item_contents, author_fallback)
child ||= PlaylistRendererParser.process(item_contents, author_fallback)
child ||= LockupViewModelParser.process(item_contents, author_fallback)
child ||= ShortsLockupViewModelParser.process(item_contents, author_fallback)
return child
end
@ -496,6 +498,9 @@ private module Parsers
# reelItemRenderer items are used in the new (2022) channel layout,
# in the "shorts" tab.
#
# NOTE: As of 10/2024, it might have been fully replaced by shortsLockupViewModel
# TODO: Confirm that hypothesis
#
module ReelItemRendererParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["reelItemRenderer"]?
@ -582,6 +587,129 @@ private module Parsers
end
end
# Parses an InnerTube lockupViewModel into a SearchPlaylist.
# Returns nil when the given object is not a lockupViewModel.
#
# This structure is present since November 2024 on the "podcasts" tab of the
# channel page. It is usually (always?) encapsulated in a richItemRenderer.
#
module LockupViewModelParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["lockupViewModel"]?
return self.parse(item_contents, author_fallback)
end
end
private def self.parse(item_contents, author_fallback)
playlist_id = item_contents["contentId"].as_s
thumbnail_view_model = item_contents.dig(
"contentImage", "collectionThumbnailViewModel",
"primaryThumbnail", "thumbnailViewModel"
)
thumbnail = thumbnail_view_model.dig("image", "sources", 1, "url").as_s
# This complicated sequences tries to extract the following data structure:
# "overlays": [{
# "thumbnailOverlayBadgeViewModel": {
# "thumbnailBadges": [{
# "thumbnailBadgeViewModel": {
# "text": "430 episodes",
# "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT"
# }
# }]
# }
# }]
video_count = thumbnail_view_model.dig("overlays").as_a
.compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a)
.flatten
.find(nil, &.dig?("thumbnailBadgeViewModel", "text").try &.as_s.ends_with?("episodes"))
.try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false)
metadata = item_contents.dig("metadata", "lockupMetadataViewModel")
title = metadata.dig("title", "content").as_s
# TODO: Retrieve "updated" info from metadata parts
# rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a
# parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s)
# One of these parts should contain a string like: "Updated 2 days ago"
# TODO: Maybe add a button to access the first video of the playlist?
# item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint")
# Available fields: "videoId", "playlistId", "params"
return SearchPlaylist.new({
title: title,
id: playlist_id,
author: author_fallback.name,
ucid: author_fallback.id,
video_count: video_count || -1,
videos: [] of SearchPlaylistVideo,
thumbnail: thumbnail,
author_verified: false,
})
end
def self.parser_name
return {{@type.name}}
end
end
# Parses an InnerTube shortsLockupViewModel into a SearchVideo.
# Returns nil when the given object is not a shortsLockupViewModel.
#
# This structure is present since around October 2024 on the "shorts" tab of
# the channel page and likely replaces the reelItemRenderer structure. It is
# usually (always?) encapsulated in a richItemRenderer.
#
module ShortsLockupViewModelParser
def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["shortsLockupViewModel"]?
return self.parse(item_contents, author_fallback)
end
end
private def self.parse(item_contents, author_fallback)
# TODO: Maybe add support for "oardefault.jpg" thumbnails?
# thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s
# Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?...
video_id = item_contents.dig(
"onTap", "innertubeCommand", "reelWatchEndpoint", "videoId"
).as_s
title = item_contents.dig("overlayMetadata", "primaryText", "content").as_s
view_count = short_text_to_number(
item_contents.dig("overlayMetadata", "secondaryText", "content").as_s
)
# Approximate to one minute, as "shorts" generally don't exceed that.
# NOTE: The actual duration is not provided by Youtube anymore.
# TODO: Maybe use -1 as an error value and handle that on the frontend?
duration = 60_i32
SearchVideo.new({
title: title,
id: video_id,
author: author_fallback.name,
ucid: author_fallback.id,
published: Time.unix(0),
views: view_count,
description_html: "",
length_seconds: duration,
premiere_timestamp: Time.unix(0),
author_verified: false,
badges: VideoBadges::None,
})
end
def self.parser_name
return {{@type.name}}
end
end
# Parses an InnerTube continuationItemRenderer into a Continuation.
# Returns nil when the given object isn't a continuationItemRenderer.
#