diff --git a/.gitignore b/.gitignore index d72439e..2bce430 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ *.dwarf data torexitnodes.txt +files +thumbnails diff --git a/config/config.yml b/config/config.example.yml similarity index 83% rename from config/config.yml rename to config/config.example.yml index 7884fbf..0a57fa0 100644 --- a/config/config.yml +++ b/config/config.example.yml @@ -1,3 +1,4 @@ +colorize_logs: true files: "./files" thumbnails: "./thumbnails" generateThumbnails: true @@ -15,6 +16,7 @@ torExitNodesCheck: 1600 torExitNodesUrl: "https://check.torproject.org/exit-addresses" torExitNodesFile: "./torexitnodes.txt" torMessage: "TOR IS BLOCKED!" +# Set this to 0 to disable rate limiting filesPerIP: 2 ipTableName: "ips" rateLimitPeriod: 20 @@ -29,7 +31,7 @@ deleteKeyLength: 4 siteInfo: "Whatever you want to put here" siteWarning: "WARNING!" log_level: "debug" - + blockedExtensions: - "exe" @@ -38,7 +40,9 @@ opengraphUseragents: - "chatterino-api-cache/" - "FFZBot/" - "Twitterbot/" + - "Synapse/" + - "Mastodon/" -alternativeDomains: - - "ayaya.beauty" - - "lamartina.gay" +# You can leave it empty, or add your own domains. +alternativeDomains: + - "example.com" diff --git a/src/config.cr b/src/config.cr index 783ad40..606c120 100644 --- a/src/config.cr +++ b/src/config.cr @@ -3,6 +3,8 @@ require "yaml" class Config include YAML::Serializable + # Colorize logs + property colorize_logs : Bool = true # Where the uploaded files will be located property files : String = "./files" # Where the thumbnails will be located when they are successfully generated @@ -12,8 +14,6 @@ class Config property generateThumbnails : Bool = false # Where the SQLITE3 database will be located property db : String = "./db.sqlite3" - # Name of the table that will be used for file information - property dbTableName : String = "files" # Enable or disable the admin API property adminEnabled : Bool = false # The API key for admin routes. It's passed as a "X-Api-Key" header to the @@ -45,8 +45,6 @@ class Config property torMessage : String? = "Tor is blocked!" # How many files an IP address can upload to the server property filesPerIP : Int32 = 32 - # Name of the table that will be used for rate limit information - property ipTableName : String = "ips" # How often is the file limit per IP reset? (in seconds) property rateLimitPeriod : Int32 = 600 # TODO: UNUSED CONSTANT diff --git a/src/file-uploader-crystal.cr b/src/file-uploader-crystal.cr index 6a293b1..eb9ea2b 100644 --- a/src/file-uploader-crystal.cr +++ b/src/file-uploader-crystal.cr @@ -18,7 +18,7 @@ Kemal.config.port = CONFIG.port Kemal.config.shutdown_message = false Kemal.config.app_name = "file-uploader-crystal" # https://github.com/iv-org/invidious/blob/90e94d4e6cc126a8b7a091d12d7a5556bfe369d5/src/invidious.cr#L136C1-L136C61 -LOGGER = LogHandler.new(STDOUT, CONFIG.log_level) +LOGGER = LogHandler.new(STDOUT, CONFIG.log_level, CONFIG.colorize_logs) # Give me a 128 bit CPU # MAX_FILES = 58**CONFIG.fileameLength SQL = DB.open("sqlite3://#{CONFIG.db}") diff --git a/src/handling/admin.cr b/src/handling/admin.cr index 5d62719..f9ec85f 100644 --- a/src/handling/admin.cr +++ b/src/handling/admin.cr @@ -17,7 +17,7 @@ module Handling::Admin file = file.to_s begin fileinfo = SQL.query_one("SELECT filename, extension, thumbnail - FROM #{CONFIG.dbTableName} + FROM files WHERE filename = ?", file, as: {filename: String, extension: String, thumbnail: String | Nil}) @@ -29,7 +29,7 @@ module Handling::Admin File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") end # Delete entry from db - SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE filename = ?", file + SQL.exec "DELETE FROM files WHERE filename = ?", file LOGGER.debug "File '#{fileinfo[:filename]}' was deleted" successfull_files << file rescue ex : DB::NoResultsError @@ -61,7 +61,7 @@ module Handling::Admin item = item.to_s begin # Delete entry from db - SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", item + SQL.exec "DELETE FROM ips WHERE ip = ?", item LOGGER.debug "Rate limit for '#{item}' was deleted" successfull << item rescue ex : DB::NoResultsError @@ -95,7 +95,7 @@ module Handling::Admin begin fileinfo = SQL.query_one("SELECT original_filename, filename, extension, uploaded_at, checksum, ip, delete_key, thumbnail - FROM #{CONFIG.dbTableName} + FROM files WHERE filename = ?", item, as: {original_filename: String, filename: String, extension: String, diff --git a/src/handling/handling.cr b/src/handling/handling.cr index 6a384f2..1b62b27 100644 --- a/src/handling/handling.cr +++ b/src/handling/handling.cr @@ -1,6 +1,7 @@ require "../http-errors" require "http/client" require "benchmark" + # require "../filters" module Handling @@ -62,11 +63,11 @@ module Handling end begin # Insert SQL data just before returning the upload information - SQL.exec "INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + SQL.exec "INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil - SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip, date) VALUES (?, ?)", ip_address, Time.utc.to_unix - # SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip) VALUES ('#{ip_address}')" - SQL.exec "UPDATE #{CONFIG.ipTableName} SET count = count + 1 WHERE ip = ('#{ip_address}')" + SQL.exec "INSERT OR IGNORE INTO ips (ip, date) VALUES (?, ?)", ip_address, Time.utc.to_unix + # SQL.exec "INSERT OR IGNORE INTO ips (ip) VALUES ('#{ip_address}')" + SQL.exec "UPDATE ips SET count = count + 1 WHERE ip = ('#{ip_address}')" rescue ex LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" return error500("An error ocurred when trying to insert the data into the DB") @@ -148,7 +149,7 @@ module Handling end begin # Insert SQL data just before returning the upload information - SQL.exec("INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil) successfull_files << {filename: filename, original_filename: original_filename, @@ -231,7 +232,7 @@ module Handling end begin # Insert SQL data just before returning the upload information - SQL.exec("INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil) successfull_files << {filename: filename, original_filename: original_filename, @@ -269,7 +270,7 @@ module Handling protocol = Utils.protocol(env) host = Utils.host(env) fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail - FROM #{CONFIG.dbTableName} + FROM files WHERE filename = ?", env.params.url["filename"].split(".").first, as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0] @@ -330,7 +331,7 @@ module Handling json.object do json.field "stats" do json.object do - json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.dbTableName}", as: Int32 + json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 json.field "maxUploadSize", CONFIG.size_limit json.field "thumbnailGeneration", CONFIG.generateThumbnails json.field "filenameLength", CONFIG.fileameLength @@ -347,10 +348,10 @@ module Handling end def delete_file(env) - if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.dbTableName} WHERE delete_key = ?)", env.params.query["key"], as: Bool + if SQL.query_one "SELECT EXISTS(SELECT 1 FROM files WHERE delete_key = ?)", env.params.query["key"], as: Bool begin fileinfo = SQL.query_all("SELECT filename, extension, thumbnail - FROM #{CONFIG.dbTableName} + FROM files WHERE delete_key = ?", env.params.query["key"], as: {filename: String, extension: String, thumbnail: String | Nil})[0] @@ -362,7 +363,7 @@ module Handling File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") end # Delete entry from db - SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE delete_key = ?", env.params.query["key"] + SQL.exec "DELETE FROM files WHERE delete_key = ?", env.params.query["key"] LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" return msg("File '#{fileinfo[:filename]}' deleted successfully") rescue ex diff --git a/src/http-errors.cr b/src/http-errors.cr index 71619b7..3023b5c 100644 --- a/src/http-errors.cr +++ b/src/http-errors.cr @@ -1,44 +1,33 @@ +macro http_error(status_code, message) + env.response.content_type = "application/json" + env.response.status_code = {{status_code}} + error_message = {"error" => {{message}}}.to_json + error_message +end + macro error400(message) - env.response.content_type = "application/json" - env.response.status_code = 400 - error_message = {"error" => {{message}}}.to_json - error_message - end + http_error(400, {{message}}) +end macro error401(message) - env.response.content_type = "application/json" - env.response.status_code = 401 - error_message = {"error" => {{message}}}.to_json - error_message - end + http_error(401, {{message}}) +end macro error403(message) - env.response.content_type = "application/json" - env.response.status_code = 403 - error_message = {"error" => {{message}}}.to_json - error_message - end + http_error(403, {{message}}) +end macro error404(message) - env.response.content_type = "application/json" - env.response.status_code = 404 - error_message = {"error" => {{message}}}.to_json - error_message - end + http_error(404, {{message}}) +end macro error413(message) - env.response.content_type = "application/json" - env.response.status_code = 413 - error_message = {"error" => {{message}}}.to_json - error_message - end + http_error(413, {{message}}) +end macro error500(message) - env.response.content_type = "application/json" - env.response.status_code = 500 - error_message = {"error" => {{message}}}.to_json - error_message - end + http_error(500, {{message}}) +end macro msg(message) env.response.content_type = "application/json" diff --git a/src/jobs.cr b/src/jobs.cr index d936e16..0698ef2 100644 --- a/src/jobs.cr +++ b/src/jobs.cr @@ -8,7 +8,7 @@ module Jobs spawn do loop do Utils.check_old_files - sleep CONFIG.deleteFilesCheck + sleep CONFIG.deleteFilesCheck.seconds end end end @@ -22,7 +22,7 @@ module Jobs Utils.retrieve_tor_exit_nodes # Updates the @@exit_nodes array instantly Routing.reload_exit_nodes - sleep CONFIG.torExitNodesCheck + sleep CONFIG.torExitNodesCheck.seconds end end end diff --git a/src/logger.cr b/src/logger.cr index dc2e5fa..b613acb 100644 --- a/src/logger.cr +++ b/src/logger.cr @@ -1,4 +1,6 @@ # https://github.com/iv-org/invidious/blob/master/src/invidious/helpers/logger.cr +require "colorize" + enum LogLevel All = 0 Trace = 1 @@ -11,7 +13,9 @@ enum LogLevel end class LogHandler < Kemal::BaseLogHandler - def initialize(@io : IO = STDOUT, @level = LogLevel::Debug) + def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true) + Colorize.enabled = use_color + Colorize.on_tty_only! end def call(context : HTTP::Server::Context) @@ -35,28 +39,27 @@ class LogHandler < Kemal::BaseLogHandler context end - def puts(message : String) - @io << message << '\n' - @io.flush - end - def write(message : String) @io << message @io.flush end - def set_log_level(level : String) - @level = LogLevel.parse(level) - end - - def set_log_level(level : LogLevel) - @level = level + def color(level) + case level + when LogLevel::Trace then :cyan + when LogLevel::Debug then :green + when LogLevel::Info then :white + when LogLevel::Warn then :yellow + when LogLevel::Error then :red + when LogLevel::Fatal then :magenta + else :default + end end {% for level in %w(trace debug info warn error fatal) %} def {{level.id}}(message : String) if LogLevel::{{level.id.capitalize}} >= @level - puts("#{Time.utc} [{{level.id}}] #{message}") + puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}}))) end end {% end %} diff --git a/src/routing.cr b/src/routing.cr index 4ddc1a5..c0931ba 100644 --- a/src/routing.cr +++ b/src/routing.cr @@ -27,14 +27,16 @@ module Routing # There is a better way to do this if env.request.resource == "/upload" begin - ip_info = SQL.query_all("SELECT ip, count, date FROM #{CONFIG.ipTableName} WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32})[0] + ip_info = SQL.query_all("SELECT ip, count, date FROM ips WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32})[0] time_since_first_upload = Time.utc.to_unix - ip_info[:date] time_until_unban = ip_info[:date] - Time.utc.to_unix + CONFIG.rateLimitPeriod if time_since_first_upload > CONFIG.rateLimitPeriod - SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", ip_info[:ip] + SQL.exec "DELETE FROM ips WHERE ip = ?", ip_info[:ip] end - if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod - halt env, status_code: 401, response: error401("Rate limited! Try again in #{time_until_unban} seconds") + if CONFIG.filesPerIP > 0 + if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod + halt env, status_code: 401, response: error401("Rate limited! Try again in #{time_until_unban} seconds") + end end rescue ex LOGGER.error "Error when trying to enforce rate limits: #{ex.message}" @@ -46,14 +48,14 @@ module Routing def register_all get "/" do |env| host = Utils.host(env) - files_hosted = SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.dbTableName}", as: Int32 + files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 render "src/views/index.ecr" end get "/chatterino" do |env| host = Utils.host(env) protocol = Utils.protocol(env) - files_hosted = SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.dbTableName}", as: Int32 + files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 render "src/views/chatterino.ecr" end diff --git a/src/utils.cr b/src/utils.cr index 0ea423e..41ddfbc 100644 --- a/src/utils.cr +++ b/src/utils.cr @@ -2,13 +2,13 @@ module Utils extend self def create_db - if !SQL.query_one "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.dbTableName}') - AND EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.ipTableName}');", as: Bool + if !SQL.query_one "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='files') + AND EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='ips');", as: Bool LOGGER.info "Creating sqlite3 database at '#{CONFIG.db}'" begin - SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.dbTableName} + SQL.exec "CREATE TABLE IF NOT EXISTS files (original_filename text, filename text, extension text, uploaded_at text, checksum text, ip text, delete_key text, thumbnail text)" - SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.ipTableName} + SQL.exec "CREATE TABLE IF NOT EXISTS ips (ip text UNIQUE, count integer DEFAULT 0, date integer)" rescue ex LOGGER.fatal "#{ex.message}" @@ -45,23 +45,23 @@ module Utils def check_old_files LOGGER.info "Deleting old files" - dir = Dir.new("#{CONFIG.files}") - # Delete entries from DB - SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE uploaded_at < date('now', '-#{CONFIG.deleteFilesAfter} days');" - # Delete files - dir.each_child do |file| - if (Time.utc - File.info("#{CONFIG.files}/#{file}").modification_time).days >= CONFIG.deleteFilesAfter - LOGGER.debug "Deleting file '#{file}'" - begin - File.delete("#{CONFIG.files}/#{file}") - rescue ex - LOGGER.error "#{ex.message}" + fileinfo = SQL.query_all("SELECT filename, extension, thumbnail + FROM files + WHERE uploaded_at < datetime('now', '-#{CONFIG.deleteFilesAfter} days')", + as: {filename: String, extension: String, thumbnail: String | Nil}) + + fileinfo.each do |file| + LOGGER.debug "Deleting file '#{file[:filename]}#{file[:extension]}'" + begin + File.delete("#{CONFIG.files}/#{file[:filename]}#{file[:extension]}") + if file[:thumbnail] + File.delete("#{CONFIG.thumbnails}/#{file[:thumbnail]}") end + SQL.exec "DELETE FROM files WHERE filename = ?", file[:filename] + rescue ex + LOGGER.error "#{ex.message}" end end - # Close directory to prevent `Too many open files (File::Error)` error. - # This is because the directory class is still saved on memory for some reason. - dir.close end def check_dependencies @@ -77,7 +77,7 @@ module Utils # TODO: # def check_duplicate(upload) - # file_checksum = SQL.query_all("SELECT checksum FROM #{CONFIG.dbTableName} WHERE original_filename = ?", upload.filename, as:String).try &.[0]? + # file_checksum = SQL.query_all("SELECT checksum FROM files WHERE original_filename = ?", upload.filename, as:String).try &.[0]? # if file_checksum.nil? # return # else @@ -101,8 +101,9 @@ module Utils # TODO: Check if there are no other possibilities to get a random filename and exit def generate_filename filename = Random.base58(CONFIG.fileameLength) + loop do - if SQL.query_one("SELECT COUNT(filename) FROM #{CONFIG.dbTableName} WHERE filename = ?", filename, as: Int32) == 0 + if SQL.query_one("SELECT COUNT(filename) FROM files WHERE filename = ?", filename, as: Int32) == 0 return filename else LOGGER.debug "Filename collision! Generating a new filename" @@ -130,8 +131,9 @@ module Utils ]) if process.exit_code == 0 LOGGER.debug "Thumbnail for #{filename + extension} generated successfully" - SQL.exec "UPDATE #{CONFIG.dbTableName} SET thumbnail = ? WHERE filename = ?", filename + ".jpg", filename + SQL.exec "UPDATE files SET thumbnail = ? WHERE filename = ?", filename + ".jpg", filename else + # TODO: Add some sort of message when the thumbnail is not generated end end @@ -159,11 +161,11 @@ module Utils # Delete file File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}") if fileinfo[:thumbnail] - # Delete thumbnail File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") end # Delete entry from db - SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE delete_key = ?", env.params.query["key"] + SQL.exec "DELETE FROM files WHERE delete_key = ?", env.params.query["key"] + LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" msg("File '#{fileinfo[:filename]}' deleted successfully") end