From cb75b97520fb30821000bff3bedfce3624e5f31f Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 21 Nov 2024 04:02:12 -0300 Subject: [PATCH] 0.9.3.3: Better handling when retrieving files, move rate limiter --- src/handling/handling.cr | 37 +++++++++++++-------------------- src/jobs.cr | 1 + src/routing.cr | 44 +++++++++++++++++++++------------------- 3 files changed, 38 insertions(+), 44 deletions(-) diff --git a/src/handling/handling.cr b/src/handling/handling.cr index 1b62b27..28e637f 100644 --- a/src/handling/handling.cr +++ b/src/handling/handling.cr @@ -42,7 +42,7 @@ module Handling return error401("Extension '#{extension}' is not allowed") end filename = Utils.generate_filename - file_path = ::File.join ["#{CONFIG.files}", filename + extension] + file_path = "#{CONFIG.files}/#{filename}#{extension}" File.open(file_path, "w") do |output| IO.copy(upload.body, output) end @@ -119,7 +119,7 @@ module Handling if CONFIG.deleteKeyLength > 0 delete_key = Random.base58(CONFIG.deleteKeyLength) end - file_path = ::File.join ["#{CONFIG.files}", filename + extension] + file_path = "#{CONFIG.files}/#{filename}#{extension}" File.open(file_path, "w") do |output| begin HTTP::Client.get(url) do |res| @@ -136,7 +136,7 @@ module Handling if extension.empty? extension = Utils.detect_extension(file_path) File.rename(file_path, file_path + extension) - file_path = ::File.join ["#{CONFIG.files}", filename + extension] + file_path = "#{CONFIG.files}/#{filename}#{extension}" end # The second one is faster and it uses less memory # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last @@ -202,7 +202,7 @@ module Handling if CONFIG.deleteKeyLength > 0 delete_key = Random.base58(CONFIG.deleteKeyLength) end - file_path = ::File.join ["#{CONFIG.files}", filename + extension] + file_path = "#{CONFIG.files}/#{filename}#{extension}" File.open(file_path, "w") do |output| begin # TODO: Connect timeout to prevent possible Denial of Service to the external website spamming requests @@ -219,7 +219,7 @@ module Handling if extension.empty? extension = Utils.detect_extension(file_path) File.rename(file_path, file_path + extension) - file_path = ::File.join ["#{CONFIG.files}", filename + extension] + file_path = "#{CONFIG.files}/#{filename}#{extension}" end # The second one is faster and it uses less memory # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last @@ -269,25 +269,16 @@ module Handling begin protocol = Utils.protocol(env) host = Utils.host(env) - fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail + fileinfo = SQL.query_one?("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail FROM files WHERE filename = ?", env.params.url["filename"].split(".").first, - as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0] - # Benchmark.ips do |x| - # x.report("header multiple") { headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}", - # "Last-Modified" => "#{fileinfo[:up_at]}", - # "ETag" => "#{fileinfo[:checksum]}"}) } - # x.report("shorter sleep") do - # env.response.headers["Content-Disposition"] = "inline; filename*=UTF-8''#{fileinfo[:ofilename]}" - # env.response.headers["Last-Modified"] = "#{fileinfo[:up_at]}" - # env.response.headers["ETag"] = "#{fileinfo[:checksum]}" - # end - # end - # `env.response.headers` is faster than `headers(env, Hash(String, String))` - # https://github.com/kemalcr/kemal/blob/3243b8e0e03568ad3bd9f0ad6f445c871605b821/src/kemal/helpers/helpers.cr#L102C1-L104C4 + as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil}) + if fileinfo.nil? + return error404("File '#{env.params.url["filename"]}' does not exist") + end env.response.headers["Content-Disposition"] = "inline; filename*=UTF-8''#{fileinfo[:ofilename]}" - # env.response.headers["Last-Modified"] = "#{fileinfo[:up_at]}" + # env.response.headers["Last-Modified"] = "#{fileinfo[:up_at]}" env.response.headers["ETag"] = "#{fileinfo[:checksum]}" CONFIG.opengraphUseragents.each do |useragent| @@ -310,8 +301,8 @@ module Handling end send_file env, "#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:ext]}" rescue ex - LOGGER.debug "File '#{env.params.url["filename"]}' does not exist: #{ex.message}" - return error403("File '#{env.params.url["filename"]}' does not exist") + LOGGER.debug "Error when retrieving file '#{env.params.url["filename"]}': #{ex.message}" + return error500("Error when retrieving file '#{env.params.url["filename"]}'") end end @@ -331,7 +322,7 @@ module Handling json.object do json.field "stats" do json.object do - json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 + json.field "filesHosted", SQL.query_one? "SELECT COUNT (filename) FROM files", as: Int32 json.field "maxUploadSize", CONFIG.size_limit json.field "thumbnailGeneration", CONFIG.generateThumbnails json.field "filenameLength", CONFIG.fileameLength diff --git a/src/jobs.cr b/src/jobs.cr index 0698ef2..ab47716 100644 --- a/src/jobs.cr +++ b/src/jobs.cr @@ -17,6 +17,7 @@ module Jobs if !CONFIG.blockTorAddresses return end + LOGGER.info("Blocking Tor exit nodes") spawn do loop do Utils.retrieve_tor_exit_nodes diff --git a/src/routing.cr b/src/routing.cr index c0931ba..2f6dd08 100644 --- a/src/routing.cr +++ b/src/routing.cr @@ -7,7 +7,7 @@ module Routing def reload_exit_nodes LOGGER.debug "Updating Tor exit nodes array" @@exit_nodes = Utils.load_tor_exit_nodes - LOGGER.debug "IPs inside the exit nodes array: #{@@exit_nodes.size}" + LOGGER.debug "IPs inside the Tor exit nodes array: #{@@exit_nodes.size}" end before_post "/api/admin/*" do |env| @@ -24,25 +24,6 @@ module Routing if CONFIG.blockTorAddresses && @@exit_nodes.includes?(Utils.ip_address(env)) halt env, status_code: 401, response: error401(CONFIG.torMessage) end - # There is a better way to do this - if env.request.resource == "/upload" - begin - ip_info = SQL.query_all("SELECT ip, count, date FROM ips WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32})[0] - time_since_first_upload = Time.utc.to_unix - ip_info[:date] - time_until_unban = ip_info[:date] - Time.utc.to_unix + CONFIG.rateLimitPeriod - if time_since_first_upload > CONFIG.rateLimitPeriod - SQL.exec "DELETE FROM ips WHERE ip = ?", ip_info[:ip] - end - if CONFIG.filesPerIP > 0 - if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod - halt env, status_code: 401, response: error401("Rate limited! Try again in #{time_until_unban} seconds") - end - end - rescue ex - LOGGER.error "Error when trying to enforce rate limits: #{ex.message}" - next - end - end end def register_all @@ -55,11 +36,32 @@ module Routing get "/chatterino" do |env| host = Utils.host(env) protocol = Utils.protocol(env) - files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 render "src/views/chatterino.ecr" end post "/upload" do |env| + begin + ip_info = SQL.query_one?("SELECT ip, count, date FROM ips WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32}) + rescue ex + LOGGER.error "Error when trying to enforce rate limits: #{ex.message}" + next + end + + if ip_info.nil? + next + end + + time_since_first_upload = Time.utc.to_unix - ip_info[:date] + time_until_unban = ip_info[:date] - Time.utc.to_unix + CONFIG.rateLimitPeriod + if time_since_first_upload > CONFIG.rateLimitPeriod + SQL.exec "DELETE FROM ips WHERE ip = ?", ip_info[:ip] + end + if CONFIG.filesPerIP > 0 + if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod + halt env, status_code: 401, response: error401("Rate limited! Try again in #{time_until_unban} seconds") + end + end + Handling.upload(env) end