From 0002c81429fceaf45a8d615d07b3be3a186d76b0 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Tue, 19 Nov 2024 22:39:23 -0300 Subject: [PATCH] 0.9.3: BUGFIX! Fix deletion of thumbnails on check_old_files job. - Add colors to logs - Use static table names instead of config provided ones, it's kinda stupid to give the user an option to set the name of the table if I'm developing it for sqlite --- .gitignore | 2 + config/{config.yml => config.example.yml} | 12 ++++-- src/config.cr | 6 +-- src/file-uploader-crystal.cr | 2 +- src/handling/admin.cr | 8 ++-- src/handling/handling.cr | 23 ++++++----- src/http-errors.cr | 49 +++++++++-------------- src/jobs.cr | 4 +- src/logger.cr | 29 ++++++++------ src/routing.cr | 14 ++++--- src/utils.cr | 48 +++++++++++----------- 11 files changed, 99 insertions(+), 98 deletions(-) rename config/{config.yml => config.example.yml} (83%) diff --git a/.gitignore b/.gitignore index d72439e..2bce430 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ *.dwarf data torexitnodes.txt +files +thumbnails diff --git a/config/config.yml b/config/config.example.yml similarity index 83% rename from config/config.yml rename to config/config.example.yml index 7884fbf..0a57fa0 100644 --- a/config/config.yml +++ b/config/config.example.yml @@ -1,3 +1,4 @@ +colorize_logs: true files: "./files" thumbnails: "./thumbnails" generateThumbnails: true @@ -15,6 +16,7 @@ torExitNodesCheck: 1600 torExitNodesUrl: "https://check.torproject.org/exit-addresses" torExitNodesFile: "./torexitnodes.txt" torMessage: "TOR IS BLOCKED!" +# Set this to 0 to disable rate limiting filesPerIP: 2 ipTableName: "ips" rateLimitPeriod: 20 @@ -29,7 +31,7 @@ deleteKeyLength: 4 siteInfo: "Whatever you want to put here" siteWarning: "WARNING!" log_level: "debug" - + blockedExtensions: - "exe" @@ -38,7 +40,9 @@ opengraphUseragents: - "chatterino-api-cache/" - "FFZBot/" - "Twitterbot/" + - "Synapse/" + - "Mastodon/" -alternativeDomains: - - "ayaya.beauty" - - "lamartina.gay" +# You can leave it empty, or add your own domains. +alternativeDomains: + - "example.com" diff --git a/src/config.cr b/src/config.cr index 783ad40..606c120 100644 --- a/src/config.cr +++ b/src/config.cr @@ -3,6 +3,8 @@ require "yaml" class Config include YAML::Serializable + # Colorize logs + property colorize_logs : Bool = true # Where the uploaded files will be located property files : String = "./files" # Where the thumbnails will be located when they are successfully generated @@ -12,8 +14,6 @@ class Config property generateThumbnails : Bool = false # Where the SQLITE3 database will be located property db : String = "./db.sqlite3" - # Name of the table that will be used for file information - property dbTableName : String = "files" # Enable or disable the admin API property adminEnabled : Bool = false # The API key for admin routes. It's passed as a "X-Api-Key" header to the @@ -45,8 +45,6 @@ class Config property torMessage : String? = "Tor is blocked!" # How many files an IP address can upload to the server property filesPerIP : Int32 = 32 - # Name of the table that will be used for rate limit information - property ipTableName : String = "ips" # How often is the file limit per IP reset? (in seconds) property rateLimitPeriod : Int32 = 600 # TODO: UNUSED CONSTANT diff --git a/src/file-uploader-crystal.cr b/src/file-uploader-crystal.cr index 6a293b1..eb9ea2b 100644 --- a/src/file-uploader-crystal.cr +++ b/src/file-uploader-crystal.cr @@ -18,7 +18,7 @@ Kemal.config.port = CONFIG.port Kemal.config.shutdown_message = false Kemal.config.app_name = "file-uploader-crystal" # https://github.com/iv-org/invidious/blob/90e94d4e6cc126a8b7a091d12d7a5556bfe369d5/src/invidious.cr#L136C1-L136C61 -LOGGER = LogHandler.new(STDOUT, CONFIG.log_level) +LOGGER = LogHandler.new(STDOUT, CONFIG.log_level, CONFIG.colorize_logs) # Give me a 128 bit CPU # MAX_FILES = 58**CONFIG.fileameLength SQL = DB.open("sqlite3://#{CONFIG.db}") diff --git a/src/handling/admin.cr b/src/handling/admin.cr index 5d62719..f9ec85f 100644 --- a/src/handling/admin.cr +++ b/src/handling/admin.cr @@ -17,7 +17,7 @@ module Handling::Admin file = file.to_s begin fileinfo = SQL.query_one("SELECT filename, extension, thumbnail - FROM #{CONFIG.dbTableName} + FROM files WHERE filename = ?", file, as: {filename: String, extension: String, thumbnail: String | Nil}) @@ -29,7 +29,7 @@ module Handling::Admin File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") end # Delete entry from db - SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE filename = ?", file + SQL.exec "DELETE FROM files WHERE filename = ?", file LOGGER.debug "File '#{fileinfo[:filename]}' was deleted" successfull_files << file rescue ex : DB::NoResultsError @@ -61,7 +61,7 @@ module Handling::Admin item = item.to_s begin # Delete entry from db - SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", item + SQL.exec "DELETE FROM ips WHERE ip = ?", item LOGGER.debug "Rate limit for '#{item}' was deleted" successfull << item rescue ex : DB::NoResultsError @@ -95,7 +95,7 @@ module Handling::Admin begin fileinfo = SQL.query_one("SELECT original_filename, filename, extension, uploaded_at, checksum, ip, delete_key, thumbnail - FROM #{CONFIG.dbTableName} + FROM files WHERE filename = ?", item, as: {original_filename: String, filename: String, extension: String, diff --git a/src/handling/handling.cr b/src/handling/handling.cr index 6a384f2..1b62b27 100644 --- a/src/handling/handling.cr +++ b/src/handling/handling.cr @@ -1,6 +1,7 @@ require "../http-errors" require "http/client" require "benchmark" + # require "../filters" module Handling @@ -62,11 +63,11 @@ module Handling end begin # Insert SQL data just before returning the upload information - SQL.exec "INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + SQL.exec "INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil - SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip, date) VALUES (?, ?)", ip_address, Time.utc.to_unix - # SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip) VALUES ('#{ip_address}')" - SQL.exec "UPDATE #{CONFIG.ipTableName} SET count = count + 1 WHERE ip = ('#{ip_address}')" + SQL.exec "INSERT OR IGNORE INTO ips (ip, date) VALUES (?, ?)", ip_address, Time.utc.to_unix + # SQL.exec "INSERT OR IGNORE INTO ips (ip) VALUES ('#{ip_address}')" + SQL.exec "UPDATE ips SET count = count + 1 WHERE ip = ('#{ip_address}')" rescue ex LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" return error500("An error ocurred when trying to insert the data into the DB") @@ -148,7 +149,7 @@ module Handling end begin # Insert SQL data just before returning the upload information - SQL.exec("INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil) successfull_files << {filename: filename, original_filename: original_filename, @@ -231,7 +232,7 @@ module Handling end begin # Insert SQL data just before returning the upload information - SQL.exec("INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil) successfull_files << {filename: filename, original_filename: original_filename, @@ -269,7 +270,7 @@ module Handling protocol = Utils.protocol(env) host = Utils.host(env) fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail - FROM #{CONFIG.dbTableName} + FROM files WHERE filename = ?", env.params.url["filename"].split(".").first, as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0] @@ -330,7 +331,7 @@ module Handling json.object do json.field "stats" do json.object do - json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.dbTableName}", as: Int32 + json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 json.field "maxUploadSize", CONFIG.size_limit json.field "thumbnailGeneration", CONFIG.generateThumbnails json.field "filenameLength", CONFIG.fileameLength @@ -347,10 +348,10 @@ module Handling end def delete_file(env) - if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.dbTableName} WHERE delete_key = ?)", env.params.query["key"], as: Bool + if SQL.query_one "SELECT EXISTS(SELECT 1 FROM files WHERE delete_key = ?)", env.params.query["key"], as: Bool begin fileinfo = SQL.query_all("SELECT filename, extension, thumbnail - FROM #{CONFIG.dbTableName} + FROM files WHERE delete_key = ?", env.params.query["key"], as: {filename: String, extension: String, thumbnail: String | Nil})[0] @@ -362,7 +363,7 @@ module Handling File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") end # Delete entry from db - SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE delete_key = ?", env.params.query["key"] + SQL.exec "DELETE FROM files WHERE delete_key = ?", env.params.query["key"] LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" return msg("File '#{fileinfo[:filename]}' deleted successfully") rescue ex diff --git a/src/http-errors.cr b/src/http-errors.cr index 71619b7..3023b5c 100644 --- a/src/http-errors.cr +++ b/src/http-errors.cr @@ -1,44 +1,33 @@ +macro http_error(status_code, message) + env.response.content_type = "application/json" + env.response.status_code = {{status_code}} + error_message = {"error" => {{message}}}.to_json + error_message +end + macro error400(message) - env.response.content_type = "application/json" - env.response.status_code = 400 - error_message = {"error" => {{message}}}.to_json - error_message - end + http_error(400, {{message}}) +end macro error401(message) - env.response.content_type = "application/json" - env.response.status_code = 401 - error_message = {"error" => {{message}}}.to_json - error_message - end + http_error(401, {{message}}) +end macro error403(message) - env.response.content_type = "application/json" - env.response.status_code = 403 - error_message = {"error" => {{message}}}.to_json - error_message - end + http_error(403, {{message}}) +end macro error404(message) - env.response.content_type = "application/json" - env.response.status_code = 404 - error_message = {"error" => {{message}}}.to_json - error_message - end + http_error(404, {{message}}) +end macro error413(message) - env.response.content_type = "application/json" - env.response.status_code = 413 - error_message = {"error" => {{message}}}.to_json - error_message - end + http_error(413, {{message}}) +end macro error500(message) - env.response.content_type = "application/json" - env.response.status_code = 500 - error_message = {"error" => {{message}}}.to_json - error_message - end + http_error(500, {{message}}) +end macro msg(message) env.response.content_type = "application/json" diff --git a/src/jobs.cr b/src/jobs.cr index d936e16..0698ef2 100644 --- a/src/jobs.cr +++ b/src/jobs.cr @@ -8,7 +8,7 @@ module Jobs spawn do loop do Utils.check_old_files - sleep CONFIG.deleteFilesCheck + sleep CONFIG.deleteFilesCheck.seconds end end end @@ -22,7 +22,7 @@ module Jobs Utils.retrieve_tor_exit_nodes # Updates the @@exit_nodes array instantly Routing.reload_exit_nodes - sleep CONFIG.torExitNodesCheck + sleep CONFIG.torExitNodesCheck.seconds end end end diff --git a/src/logger.cr b/src/logger.cr index dc2e5fa..b613acb 100644 --- a/src/logger.cr +++ b/src/logger.cr @@ -1,4 +1,6 @@ # https://github.com/iv-org/invidious/blob/master/src/invidious/helpers/logger.cr +require "colorize" + enum LogLevel All = 0 Trace = 1 @@ -11,7 +13,9 @@ enum LogLevel end class LogHandler < Kemal::BaseLogHandler - def initialize(@io : IO = STDOUT, @level = LogLevel::Debug) + def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true) + Colorize.enabled = use_color + Colorize.on_tty_only! end def call(context : HTTP::Server::Context) @@ -35,28 +39,27 @@ class LogHandler < Kemal::BaseLogHandler context end - def puts(message : String) - @io << message << '\n' - @io.flush - end - def write(message : String) @io << message @io.flush end - def set_log_level(level : String) - @level = LogLevel.parse(level) - end - - def set_log_level(level : LogLevel) - @level = level + def color(level) + case level + when LogLevel::Trace then :cyan + when LogLevel::Debug then :green + when LogLevel::Info then :white + when LogLevel::Warn then :yellow + when LogLevel::Error then :red + when LogLevel::Fatal then :magenta + else :default + end end {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) if LogLevel::{{level.id.capitalize}} >= @level - puts("#{Time.utc} [{{level.id}}] #{message}") + puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}}))) end end {% end %} diff --git a/src/routing.cr b/src/routing.cr index 4ddc1a5..c0931ba 100644 --- a/src/routing.cr +++ b/src/routing.cr @@ -27,14 +27,16 @@ module Routing # There is a better way to do this if env.request.resource == "/upload" begin - ip_info = SQL.query_all("SELECT ip, count, date FROM #{CONFIG.ipTableName} WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32})[0] + ip_info = SQL.query_all("SELECT ip, count, date FROM ips WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32})[0] time_since_first_upload = Time.utc.to_unix - ip_info[:date] time_until_unban = ip_info[:date] - Time.utc.to_unix + CONFIG.rateLimitPeriod if time_since_first_upload > CONFIG.rateLimitPeriod - SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", ip_info[:ip] + SQL.exec "DELETE FROM ips WHERE ip = ?", ip_info[:ip] end - if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod - halt env, status_code: 401, response: error401("Rate limited! Try again in #{time_until_unban} seconds") + if CONFIG.filesPerIP > 0 + if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod + halt env, status_code: 401, response: error401("Rate limited! Try again in #{time_until_unban} seconds") + end end rescue ex LOGGER.error "Error when trying to enforce rate limits: #{ex.message}" @@ -46,14 +48,14 @@ module Routing def register_all get "/" do |env| host = Utils.host(env) - files_hosted = SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.dbTableName}", as: Int32 + files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 render "src/views/index.ecr" end get "/chatterino" do |env| host = Utils.host(env) protocol = Utils.protocol(env) - files_hosted = SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.dbTableName}", as: Int32 + files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 render "src/views/chatterino.ecr" end diff --git a/src/utils.cr b/src/utils.cr index 0ea423e..41ddfbc 100644 --- a/src/utils.cr +++ b/src/utils.cr @@ -2,13 +2,13 @@ module Utils extend self def create_db - if !SQL.query_one "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.dbTableName}') - AND EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.ipTableName}');", as: Bool + if !SQL.query_one "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='files') + AND EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='ips');", as: Bool LOGGER.info "Creating sqlite3 database at '#{CONFIG.db}'" begin - SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.dbTableName} + SQL.exec "CREATE TABLE IF NOT EXISTS files (original_filename text, filename text, extension text, uploaded_at text, checksum text, ip text, delete_key text, thumbnail text)" - SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.ipTableName} + SQL.exec "CREATE TABLE IF NOT EXISTS ips (ip text UNIQUE, count integer DEFAULT 0, date integer)" rescue ex LOGGER.fatal "#{ex.message}" @@ -45,23 +45,23 @@ module Utils def check_old_files LOGGER.info "Deleting old files" - dir = Dir.new("#{CONFIG.files}") - # Delete entries from DB - SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE uploaded_at < date('now', '-#{CONFIG.deleteFilesAfter} days');" - # Delete files - dir.each_child do |file| - if (Time.utc - File.info("#{CONFIG.files}/#{file}").modification_time).days >= CONFIG.deleteFilesAfter - LOGGER.debug "Deleting file '#{file}'" - begin - File.delete("#{CONFIG.files}/#{file}") - rescue ex - LOGGER.error "#{ex.message}" + fileinfo = SQL.query_all("SELECT filename, extension, thumbnail + FROM files + WHERE uploaded_at < datetime('now', '-#{CONFIG.deleteFilesAfter} days')", + as: {filename: String, extension: String, thumbnail: String | Nil}) + + fileinfo.each do |file| + LOGGER.debug "Deleting file '#{file[:filename]}#{file[:extension]}'" + begin + File.delete("#{CONFIG.files}/#{file[:filename]}#{file[:extension]}") + if file[:thumbnail] + File.delete("#{CONFIG.thumbnails}/#{file[:thumbnail]}") end + SQL.exec "DELETE FROM files WHERE filename = ?", file[:filename] + rescue ex + LOGGER.error "#{ex.message}" end end - # Close directory to prevent `Too many open files (File::Error)` error. - # This is because the directory class is still saved on memory for some reason. - dir.close end def check_dependencies @@ -77,7 +77,7 @@ module Utils # TODO: # def check_duplicate(upload) - # file_checksum = SQL.query_all("SELECT checksum FROM #{CONFIG.dbTableName} WHERE original_filename = ?", upload.filename, as:String).try &.[0]? + # file_checksum = SQL.query_all("SELECT checksum FROM files WHERE original_filename = ?", upload.filename, as:String).try &.[0]? # if file_checksum.nil? # return # else @@ -101,8 +101,9 @@ module Utils # TODO: Check if there are no other possibilities to get a random filename and exit def generate_filename filename = Random.base58(CONFIG.fileameLength) + loop do - if SQL.query_one("SELECT COUNT(filename) FROM #{CONFIG.dbTableName} WHERE filename = ?", filename, as: Int32) == 0 + if SQL.query_one("SELECT COUNT(filename) FROM files WHERE filename = ?", filename, as: Int32) == 0 return filename else LOGGER.debug "Filename collision! Generating a new filename" @@ -130,8 +131,9 @@ module Utils ]) if process.exit_code == 0 LOGGER.debug "Thumbnail for #{filename + extension} generated successfully" - SQL.exec "UPDATE #{CONFIG.dbTableName} SET thumbnail = ? WHERE filename = ?", filename + ".jpg", filename + SQL.exec "UPDATE files SET thumbnail = ? WHERE filename = ?", filename + ".jpg", filename else + # TODO: Add some sort of message when the thumbnail is not generated end end @@ -159,11 +161,11 @@ module Utils # Delete file File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}") if fileinfo[:thumbnail] - # Delete thumbnail File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") end # Delete entry from db - SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE delete_key = ?", env.params.query["key"] + SQL.exec "DELETE FROM files WHERE delete_key = ?", env.params.query["key"] + LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" msg("File '#{fileinfo[:filename]}' deleted successfully") end