diff --git a/config/config.yml b/config/config.yml index c24bc36..7884fbf 100644 --- a/config/config.yml +++ b/config/config.yml @@ -1,6 +1,6 @@ files: "./files" thumbnails: "./thumbnails" -generateThumbnails: false +generateThumbnails: true db: "./db.sqlite3" dbTableName: "files" adminEnabled: true @@ -11,8 +11,8 @@ size_limit: 512 port: 8080 blockTorAddresses: true # Every hour -torExitNodesCheck: 3600 -torExitNodesUrl: "https://www.dan.me.uk/torlist/?exit" +torExitNodesCheck: 1600 +torExitNodesUrl: "https://check.torproject.org/exit-addresses" torExitNodesFile: "./torexitnodes.txt" torMessage: "TOR IS BLOCKED!" filesPerIP: 2 @@ -22,9 +22,9 @@ rateLimitMessage: "" # If you define the unix socket, it will only listen on the socket and not the port. #unix_socket: "/tmp/file-uploader.sock" # In days -deleteFilesAfter: 7 +deleteFilesAfter: 1 # In seconds -deleteFilesCheck: 1800 +deleteFilesCheck: 1600 deleteKeyLength: 4 siteInfo: "Whatever you want to put here" siteWarning: "WARNING!" diff --git a/src/config.cr b/src/config.cr index c94643d..783ad40 100644 --- a/src/config.cr +++ b/src/config.cr @@ -35,9 +35,8 @@ class Config property blockTorAddresses : Bool? = false # How often (in seconds) should this program download the exit nodes list property torExitNodesCheck : Int32 = 3600 - # A URL with a list of exit nodes addresses - # The list needs to contain a IP address per line - property torExitNodesUrl : String = "https://www.dan.me.uk/torlist/?exit" + # Only https://check.torproject.org/exit-addresses is supported + property torExitNodesUrl : String = "https://check.torproject.org/exit-addresses" # Where the file of the exit nodes will be located, can be placed anywhere property torExitNodesFile : String = "./torexitnodes.txt" # Message that will be displayed to the Tor user. diff --git a/src/handling/admin.cr b/src/handling/admin.cr index e3ad075..7a53526 100644 --- a/src/handling/admin.cr +++ b/src/handling/admin.cr @@ -3,7 +3,12 @@ require "../http-errors" module Handling::Admin extend self + # private macro json_fill(named_tuple, field_name) + # j.field {{field_name}}, {{named_tuple}}[:{{field_name}}] + # end + # /api/admin/delete + # curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: asd" http://localhost:8080/api/admin/delete -d '{"files": ["j63"]}' | jq def delete_file(env) files = env.params.json["files"].as((Array(JSON::Any))) successfull_files = [] of String @@ -46,20 +51,22 @@ module Handling::Admin end # /api/admin/deleteiplimit + # curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: asd" http://localhost:8080/api/admin/deleteiplimit -d '{"ips": ["127.0.0.1"]}' | jq + def delete_ip_limit(env) - ips = env.params.json["ips"].as((Array(JSON::Any))) - successfull_ips = [] of String - failed_ips = [] of String - ips.each do |ip| - ip = ip.to_s + data = env.params.json["ips"].as((Array(JSON::Any))) + successfull = [] of String + failed = [] of String + data.each do |item| + item = item.to_s begin # Delete entry from db - SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", ip - LOGGER.debug "Rate limit for '#{ip}' was deleted" - successfull_ips << ip + SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", item + LOGGER.debug "Rate limit for '#{item}' was deleted" + successfull << item rescue ex : DB::NoResultsError - LOGGER.error("Rate limit for '#{ip}' doesn't exist or is not registered in the database: #{ex.message}") - failed_ips << ip + LOGGER.error("Rate limit for '#{item}' doesn't exist or is not registered in the database: #{ex.message}") + failed << item rescue ex LOGGER.error "Unknown error: #{ex.message}" error500 "Unknown error: #{ex.message}" @@ -67,11 +74,101 @@ module Handling::Admin end json = JSON.build do |j| j.object do - j.field "successfull", successfull_ips.size - j.field "failed", failed_ips.size - j.field "successfullUnbans", successfull_ips - j.field "failedUnbans", failed_ips + j.field "successfull", successfull.size + j.field "failed", failed.size + j.field "successfullUnbans", successfull + j.field "failedUnbans", failed end end end + + # /api/admin/fileinfo + # curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: asd" http://localhost:8080/api/admin/fileinfo -d '{"files": ["j63"]}' | jq + def retrieve_file_info(env) + data = env.params.json["files"].as((Array(JSON::Any))) + successfull = [] of NamedTuple(original_filename: String, filename: String, extension: String, + uploaded_at: String, checksum: String, ip: String, delete_key: String, + thumbnail: String | Nil) + failed = [] of String + data.each do |item| + item = item.to_s + begin + fileinfo = SQL.query_one("SELECT original_filename, filename, extension, + uploaded_at, checksum, ip, delete_key, thumbnail + FROM #{CONFIG.dbTableName} + WHERE filename = ?", + item, + as: {original_filename: String, filename: String, extension: String, + uploaded_at: String, checksum: String, ip: String, delete_key: String, + thumbnail: String | Nil}) + successfull << fileinfo + rescue ex : DB::NoResultsError + LOGGER.error("File '#{item}' is not registered in the database: #{ex.message}") + failed << item + rescue ex + LOGGER.error "Unknown error: #{ex.message}" + error500 "Unknown error: #{ex.message}" + end + end + json = JSON.build do |j| + j.object do + j.field "files" do + j.array do + successfull.each do |fileinfo| + j.object do + j.field fileinfo[:filename] do + j.object do + j.field "original_filename", fileinfo[:original_filename] + j.field "filename", fileinfo[:filename] + j.field "extension", fileinfo[:extension] + j.field "uploaded_at", fileinfo[:uploaded_at] + j.field "checksum", fileinfo[:checksum] + j.field "ip", fileinfo[:ip] + j.field "delete_key", fileinfo[:delete_key] + j.field "thumbnail", fileinfo[:thumbnail] + end + end + end + end + end + end + j.field "successfull", successfull.size + j.field "failed", failed.size + # j.field "successfullFiles" + j.field "failedFiles", failed + end + end + end + + # /api/admin/torexitnodes + # curl -X GET -H "X-Api-Key: asd" http://localhost:8080/api/admin/torexitnodes | jq + def retrieve_tor_exit_nodes(env, nodes) + json = JSON.build do |j| + j.object do + j.field "ips", nodes + end + end + end + + # /api/admin/whitelist + # curl -X GET -H "X-Api-Key: asd" http://localhost:8080/api/admin/torexitnodes | jq +# def add_ip_to_whitelist(env, nodes) +# json = JSON.build do |j| +# j.object do +# j.field "ips", nodes +# end +# end +# end + + # /api/admin/blacklist + # curl -X GET -H "X-Api-Key: asd" http://localhost:8080/api/admin/torexitnodes | jq + def add_ip_to_blacklist(env, nodes) + json = JSON.build do |j| + j.object do + j.field "ips", nodes + end + end + end + + # MODULE END end diff --git a/src/handling/handling.cr b/src/handling/handling.cr index f5f3c21..13260d5 100644 --- a/src/handling/handling.cr +++ b/src/handling/handling.cr @@ -1,5 +1,6 @@ require "../http-errors" require "http/client" +require "benchmark" module Handling extend self @@ -43,7 +44,7 @@ module Handling IO.copy(upload.body, output) end original_filename = upload.filename - uploaded_at = Time::Format::HTTP_DATE.format(Time.utc) + uploaded_at = Time.utc checksum = Utils.hash_file(file_path) end # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse @@ -101,7 +102,7 @@ module Handling original_filename = "" extension = "" checksum = "" - uploaded_at = Time::Format::HTTP_DATE.format(Time.utc) + uploaded_at = Time.utc extension = File.extname(URI.parse(url).path) delete_key = nil file_path = ::File.join ["#{CONFIG.files}", filename + extension] @@ -123,7 +124,7 @@ module Handling File.rename(file_path, file_path + extension) file_path = ::File.join ["#{CONFIG.files}", filename + extension] end - # TODO: Benchmark this: + # The second one is faster and it uses less memory # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last original_filename = url.split("/").last checksum = Utils.hash_file(file_path) @@ -177,10 +178,21 @@ module Handling WHERE filename = ?", env.params.url["filename"].split(".").first, as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0] - - headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"}) - headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"}) - headers(env, {"ETag" => "#{fileinfo[:checksum]}"}) + # Benchmark.ips do |x| + # x.report("header multiple") { headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}", + # "Last-Modified" => "#{fileinfo[:up_at]}", + # "ETag" => "#{fileinfo[:checksum]}"}) } + # x.report("shorter sleep") do + # env.response.headers["Content-Disposition"] = "inline; filename*=UTF-8''#{fileinfo[:ofilename]}" + # env.response.headers["Last-Modified"] = "#{fileinfo[:up_at]}" + # env.response.headers["ETag"] = "#{fileinfo[:checksum]}" + # end + # end + # `env.response.headers` is faster than `headers(env, Hash(String, String))` + # https://github.com/kemalcr/kemal/blob/3243b8e0e03568ad3bd9f0ad6f445c871605b821/src/kemal/helpers/helpers.cr#L102C1-L104C4 + env.response.headers["Content-Disposition"] = "inline; filename*=UTF-8''#{fileinfo[:ofilename]}" + # env.response.headers["Last-Modified"] = "#{fileinfo[:up_at]}" + env.response.headers["ETag"] = "#{fileinfo[:checksum]}" CONFIG.opengraphUseragents.each do |useragent| if env.request.headers.try &.["User-Agent"].includes?(useragent) diff --git a/src/jobs.cr b/src/jobs.cr index f24a5f9..d936e16 100644 --- a/src/jobs.cr +++ b/src/jobs.cr @@ -20,8 +20,8 @@ module Jobs spawn do loop do Utils.retrieve_tor_exit_nodes - # Updates the @@exit_nodes array instantly - Routing.reload_exit_nodes + # Updates the @@exit_nodes array instantly + Routing.reload_exit_nodes sleep CONFIG.torExitNodesCheck end end diff --git a/src/routing.cr b/src/routing.cr index 6e7b2d8..384d350 100644 --- a/src/routing.cr +++ b/src/routing.cr @@ -7,6 +7,7 @@ module Routing def reload_exit_nodes LOGGER.debug "Updating Tor exit nodes array" @@exit_nodes = Utils.load_tor_exit_nodes + LOGGER.debug "IPs inside the exit nodes array: #{@@exit_nodes.size}" end before_post "/api/admin/*" do |env| @@ -94,4 +95,12 @@ module Routing post "/api/admin/deleteiplimit" do |env| Handling::Admin.delete_ip_limit(env) end + + post "/api/admin/fileinfo" do |env| + Handling::Admin.retrieve_file_info(env) + end + + get "/api/admin/torexitnodes" do |env| + Handling::Admin.retrieve_tor_exit_nodes(env, @@exit_nodes) + end end diff --git a/src/utils.cr b/src/utils.cr index 879412a..0c87bf7 100644 --- a/src/utils.cr +++ b/src/utils.cr @@ -181,21 +181,21 @@ module Utils ".wmv" => "󠀀3026b2758e66cf11", ".flv" => "󠀀464c5601", ".mpeg" => "000001bx", - # Audio - ".mp3" => "󠀀494433", - ".aac" => "󠀀fff1", - ".wav" => "󠀀57415645666d7420", + # Audio + ".mp3" => "󠀀494433", + ".aac" => "󠀀fff1", + ".wav" => "󠀀57415645666d7420", ".flac" => "󠀀664c614300000022", - ".ogg" => "󠀀4f67675300020000000000000000", - ".wma" => "󠀀3026b2758e66cf11a6d900aa0062ce6c", + ".ogg" => "󠀀4f67675300020000000000000000", + ".wma" => "󠀀3026b2758e66cf11a6d900aa0062ce6c", ".aiff" => "󠀀464f524d00", # Whatever ".7z" => "377abcaf271c", ".gz" => "1f8b", ".iso" => "󠀀4344303031", - # Documents - "pdf" => "󠀀25504446", - "html" => "", + # Documents + "pdf" => "󠀀25504446", + "html" => "", } def detect_extension(file) : String @@ -235,6 +235,13 @@ module Utils def load_tor_exit_nodes exit_nodes = File.read_lines(CONFIG.torExitNodesFile) + ips = [] of String + exit_nodes.each do |line| + if line.includes?("ExitAddress") + ips << line.split(" ")[1] + end + end + return ips end def ip_address(env) : String