Fijxu/file-uploader-crystal
1
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages 1 Wiki Activity Actions

0.9.3.3: Better handling when retrieving files, move rate limiter
All checks were successful
File-uploader-crystal CI / build (push) Successful in 1m47s
Details

Browse source
This commit is contained in:
Fijxu 2024-11-21 04:02:12 -03:00
parent fdfa782e91
commit cb75b97520
Signed by: Fijxu
GPG key ID: 32C1DDF333EDA6A4
3 changed files with 38 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
src/handling/handling.cr
View file

@ -42,7 +42,7 @@ module Handling
return error401("Extension '#{extension}' is not allowed")
end
filename = Utils.generate_filename
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
file_path = "#{CONFIG.files}/#{filename}#{extension}"
File.open(file_path, "w") do |output|
IO.copy(upload.body, output)
end
@ -119,7 +119,7 @@ module Handling
if CONFIG.deleteKeyLength > 0
delete_key = Random.base58(CONFIG.deleteKeyLength)
end
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
file_path = "#{CONFIG.files}/#{filename}#{extension}"
File.open(file_path, "w") do |output|
begin
HTTP::Client.get(url) do |res|
@ -136,7 +136,7 @@ module Handling
if extension.empty?
extension = Utils.detect_extension(file_path)
File.rename(file_path, file_path + extension)
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
file_path = "#{CONFIG.files}/#{filename}#{extension}"
end
# The second one is faster and it uses less memory
# original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last
@ -202,7 +202,7 @@ module Handling
if CONFIG.deleteKeyLength > 0
delete_key = Random.base58(CONFIG.deleteKeyLength)
end
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
file_path = "#{CONFIG.files}/#{filename}#{extension}"
File.open(file_path, "w") do |output|
begin
# TODO: Connect timeout to prevent possible Denial of Service to the external website spamming requests
@ -219,7 +219,7 @@ module Handling
if extension.empty?
extension = Utils.detect_extension(file_path)
File.rename(file_path, file_path + extension)
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
file_path = "#{CONFIG.files}/#{filename}#{extension}"
end
# The second one is faster and it uses less memory
# original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last
@ -269,25 +269,16 @@ module Handling
begin
protocol = Utils.protocol(env)
host = Utils.host(env)
fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail
fileinfo = SQL.query_one?("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail
FROM files
WHERE filename = ?",
env.params.url["filename"].split(".").first,
as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0]
# Benchmark.ips do |x|
# x.report("header multiple") { headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}",
# "Last-Modified" => "#{fileinfo[:up_at]}",
# "ETag" => "#{fileinfo[:checksum]}"}) }
# x.report("shorter sleep") do
# env.response.headers["Content-Disposition"] = "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"
# env.response.headers["Last-Modified"] = "#{fileinfo[:up_at]}"
# env.response.headers["ETag"] = "#{fileinfo[:checksum]}"
# end
# end
# `env.response.headers` is faster than `headers(env, Hash(String, String))`
# https://github.com/kemalcr/kemal/blob/3243b8e0e03568ad3bd9f0ad6f445c871605b821/src/kemal/helpers/helpers.cr#L102C1-L104C4
as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})
if fileinfo.nil?
return error404("File '#{env.params.url["filename"]}' does not exist")
end
env.response.headers["Content-Disposition"] = "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"
# env.response.headers["Last-Modified"] = "#{fileinfo[:up_at]}"
# env.response.headers["Last-Modified"] = "#{fileinfo[:up_at]}"
env.response.headers["ETag"] = "#{fileinfo[:checksum]}"
CONFIG.opengraphUseragents.each do |useragent|
@ -310,8 +301,8 @@ module Handling
end
send_file env, "#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:ext]}"
rescue ex
LOGGER.debug "File '#{env.params.url["filename"]}' does not exist: #{ex.message}"
return error403("File '#{env.params.url["filename"]}' does not exist")
LOGGER.debug "Error when retrieving file '#{env.params.url["filename"]}': #{ex.message}"
return error500("Error when retrieving file '#{env.params.url["filename"]}'")
end
end
@ -331,7 +322,7 @@ module Handling
json.object do
json.field "stats" do
json.object do
json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32
json.field "filesHosted", SQL.query_one? "SELECT COUNT (filename) FROM files", as: Int32
json.field "maxUploadSize", CONFIG.size_limit
json.field "thumbnailGeneration", CONFIG.generateThumbnails
json.field "filenameLength", CONFIG.fileameLength

1
src/jobs.cr
View file

@ -17,6 +17,7 @@ module Jobs
if !CONFIG.blockTorAddresses
return
end
LOGGER.info("Blocking Tor exit nodes")
spawn do
loop do
Utils.retrieve_tor_exit_nodes

44
src/routing.cr
View file

@ -7,7 +7,7 @@ module Routing
def reload_exit_nodes
LOGGER.debug "Updating Tor exit nodes array"
@@exit_nodes = Utils.load_tor_exit_nodes
LOGGER.debug "IPs inside the exit nodes array: #{@@exit_nodes.size}"
LOGGER.debug "IPs inside the Tor exit nodes array: #{@@exit_nodes.size}"
end
before_post "/api/admin/*" do |env|
@ -24,25 +24,6 @@ module Routing
if CONFIG.blockTorAddresses && @@exit_nodes.includes?(Utils.ip_address(env))
halt env, status_code: 401, response: error401(CONFIG.torMessage)
end
# There is a better way to do this
if env.request.resource == "/upload"
begin
ip_info = SQL.query_all("SELECT ip, count, date FROM ips WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32})[0]
time_since_first_upload = Time.utc.to_unix - ip_info[:date]
time_until_unban = ip_info[:date] - Time.utc.to_unix + CONFIG.rateLimitPeriod
if time_since_first_upload > CONFIG.rateLimitPeriod
SQL.exec "DELETE FROM ips WHERE ip = ?", ip_info[:ip]
end
if CONFIG.filesPerIP > 0
if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod
halt env, status_code: 401, response: error401("Rate limited! Try again in #{time_until_unban} seconds")
end
end
rescue ex
LOGGER.error "Error when trying to enforce rate limits: #{ex.message}"
next
end
end
end
def register_all
@ -55,11 +36,32 @@ module Routing
get "/chatterino" do |env|
host = Utils.host(env)
protocol = Utils.protocol(env)
files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32
render "src/views/chatterino.ecr"
end
post "/upload" do |env|
begin
ip_info = SQL.query_one?("SELECT ip, count, date FROM ips WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32})
rescue ex
LOGGER.error "Error when trying to enforce rate limits: #{ex.message}"
next
end
if ip_info.nil?
next
end
time_since_first_upload = Time.utc.to_unix - ip_info[:date]
time_until_unban = ip_info[:date] - Time.utc.to_unix + CONFIG.rateLimitPeriod
if time_since_first_upload > CONFIG.rateLimitPeriod
SQL.exec "DELETE FROM ips WHERE ip = ?", ip_info[:ip]
end
if CONFIG.filesPerIP > 0
if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod
halt env, status_code: 401, response: error401("Rate limited! Try again in #{time_until_unban} seconds")
end
end
Handling.upload(env)
end