diff --git a/src/invidious.cr b/src/invidious.cr index 8f0a482c..e1bf18b9 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -156,19 +156,6 @@ end OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a") LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level, CONFIG.colorize_logs) -REDIS_DB = begin - LOGGER.info "Connecting to Redis compatible DB" - redis = Redis::PooledClient.new(unixsocket: CONFIG.redis_socket || nil, url: CONFIG.redis_url || nil) - if redis.ping - LOGGER.info "Connected to Redis compatible DB via unix domain socket at '#{CONFIG.redis_socket}'" if CONFIG.redis_socket - LOGGER.info "Connected to Redis compatible DB via TCP socket at '#{CONFIG.redis_url}'" if CONFIG.redis_url - end - redis -rescue ex - LOGGER.error "Failed to connect to a Redis compatible DB. Invidious will store the video cache on the PostgresSQL DB" - nil -end - # Check table integrity Invidious::Database.check_integrity(CONFIG) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 4baa0dcb..e1a06f85 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -217,7 +217,16 @@ class Config property tokens_server : String = "" - property video_cache : Bool = true + property video_cache : VideoCacheConfig + + class VideoCacheConfig + include YAML::Serializable + + property enabled : Bool = true + property backend : Int32 = 1 + # Max quantity of keys that can be held on the LRU cache + property lru_max_size : Int32 = 18432 # ~512MB + end {% if flag?(:linux) %} property reload_config_automatically : Bool = true @@ -395,6 +404,16 @@ class Config end end + if config.video_cache.enabled + if !config.video_cache.backend.in?(0, 1, 2) + puts "Config: 'video_cache_storage', can only be:" + puts "0 (PostgreSQL)" + puts "1 (Redis compatible DB) (Default)" + puts "2 (In memory LRU)" + exit(1) + end + end + return config end end diff --git a/src/invidious/database/videos.cr b/src/invidious/database/videos.cr index e826ae79..33d0a100 100644 --- a/src/invidious/database/videos.cr +++ b/src/invidious/database/videos.cr @@ -1,42 +1,63 @@ require "./base.cr" +require "redis" + +VideoCache = Invidious::Database::Videos::Cache.new module Invidious::Database::Videos - module DBCache - extend self + class Cache + def initialize + case CONFIG.video_cache.backend + when 0 + @cache = CacheMethods::PostgresSQL.new + when 1 + @cache = CacheMethods::Redis_.new + when 2 + @cache = CacheMethods::LRU.new + else + LOGGER.debug "Video Cache: Using default cache method to store video cache (PostgreSQL)" + @cache = CacheMethods::PostgresSQL.new + end + end def set(video : Video, expire_time) - if redis = REDIS_DB - redis.set(video.id, video.info.to_json, expire_time) - redis.set(video.id + ":time", video.updated, expire_time) - else - request = <<-SQL - INSERT INTO videos - VALUES ($1, $2, $3) - ON CONFLICT (id) DO NOTHING - SQL - - PG_DB.exec(request, video.id, video.info.to_json, video.updated) - end + @cache.set(video, expire_time) end def del(id : String) - if redis = REDIS_DB - redis.del(id) - redis.del(id + ":time") - else - request = <<-SQL - DELETE FROM videos * - WHERE id = $1 - SQL - - PG_DB.exec(request, id) - end + @cache.del(id) end def get(id : String) - if redis = REDIS_DB - info = redis.get(id) - time = redis.get(id + ":time") + return @cache.get(id) + end + end + + module CacheMethods + # TODO: Save the cache on a file with a Job + class LRU + @max_size : Int32 + @lru = {} of String => String + @access = [] of String + + def initialize(@max_size = CONFIG.video_cache.lru_max_size) + LOGGER.info "Video Cache: Using in memory LRU to store video cache" + LOGGER.info "Video Cache, LRU: LRU cache max size set to #{@max_size}" + end + + # TODO: Handle expire_time with a Job + def set(video : Video, expire_time) + self[video.id] = video.info.to_json + self[video.id + ":time"] = "#{video.updated}" + end + + def del(id : String) + self.delete(id) + self.delete(id + ":time") + end + + def get(id : String) + info = self[id] + time = self[id + ":time"] if info && time return Video.new({ id: id, @@ -46,7 +67,98 @@ module Invidious::Database::Videos else return nil end - else + end + + private def [](key) + if @lru[key]? + @access.delete(key) + @access.push(key) + @lru[key] + else + nil + end + end + + private def []=(key, value) + if @lru.size >= @max_size + lru_key = @access.shift + @lru.delete(lru_key) + end + @lru[key] = value + @access.push(key) + end + + private def delete(key) + if @lru[key]? + @lru.delete(key) + @access.delete(key) + end + end + end + + class Redis_ + @redis : Redis::PooledClient + + def initialize + @redis = Redis::PooledClient.new(unixsocket: CONFIG.redis_socket || nil, url: CONFIG.redis_url || nil) + LOGGER.info "Video Cache: Using Redis compatible DB to store video cache" + LOGGER.info "Connecting to Redis compatible DB" + if @redis.ping + LOGGER.info "Connected to Redis compatible DB via unix domain socket at '#{CONFIG.redis_socket}'" if CONFIG.redis_socket + LOGGER.info "Connected to Redis compatible DB via TCP socket at '#{CONFIG.redis_url}'" if CONFIG.redis_url + end + end + + def set(video : Video, expire_time) + @redis.set(video.id, video.info.to_json, expire_time) + @redis.set(video.id + ":time", video.updated, expire_time) + end + + def del(id : String) + @redis.del(id) + @redis.del(id + ":time") + end + + def get(id : String) + info = @redis.get(id) + time = @redis.get(id + ":time") + if info && time + return Video.new({ + id: id, + info: JSON.parse(info).as_h, + updated: Time.parse(time, "%Y-%m-%d %H:%M:%S %z", Time::Location::UTC), + }) + else + return nil + end + end + end + + class PostgresSQL + def initialize + LOGGER.info "Video Cache: Using PostgreSQL to store video cache" + end + + def set(video : Video, expire_time) + request = <<-SQL + INSERT INTO videos + VALUES ($1, $2, $3) + ON CONFLICT (id) DO NOTHING + SQL + + PG_DB.exec(request, video.id, video.info.to_json, video.updated) + end + + def del(id) + request = <<-SQL + DELETE FROM videos * + WHERE id = $1 + SQL + + PG_DB.exec(request, id) + end + + def get(id : String) : Video? request = <<-SQL SELECT * FROM videos WHERE id = $1 @@ -60,11 +172,11 @@ module Invidious::Database::Videos extend self def insert(video : Video) - DBCache.set(video: video, expire_time: 14400) if CONFIG.video_cache + VideoCache.set(video: video, expire_time: 14400) if CONFIG.video_cache.enabled end def delete(id) - DBCache.del(id) + VideoCache.del(id) end def delete_expired @@ -87,6 +199,6 @@ module Invidious::Database::Videos end def select(id : String) : Video? - return DBCache.get(id) + return VideoCache.get(id) end end