diff --git a/README.md b/README.md index fac69fc..efbb088 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Already replaced lol. - File deletion link (not available in frontend for now) - Chatterino and ShareX support - Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, can be disabled.) +- Rate Limiting (WIP) - [Small Admin API](./src/handling/admin.cr) that allows you to delete files. (Needs to be enabled in the configuration) - Unix socket support if you don't want to deal with all the TCP overhead - Automatic protocol detection (HTTPS or HTTP) diff --git a/shard.yml b/shard.yml index b80f141..a800b4f 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: file-uploader -version: 0.8.0 +version: 0.8.7 authors: - Fijxu diff --git a/src/config.cr b/src/config.cr index 9280e42..4d9ae49 100644 --- a/src/config.cr +++ b/src/config.cr @@ -22,6 +22,10 @@ class Config # The list needs to contain a IP address per line property torExitNodesUrl : String = "https://www.dan.me.uk/torlist/?exit" property torExitNodesFile : String = "./torexitnodes.txt" + property filesPerIP : Int32 = 32 + property ipTableName : String = "ips" + # How often is the file limit per IP reset? + property rateLimitPeriod : Int32 = 600 property torMessage : String? = "Tor is blocked!" property deleteFilesAfter : Int32 = 7 # How often should the check of old files be performed? (in seconds) diff --git a/src/handling/handling.cr b/src/handling/handling.cr index a15e8fd..d408f71 100644 --- a/src/handling/handling.cr +++ b/src/handling/handling.cr @@ -6,12 +6,15 @@ module Handling def upload(env) env.response.content_type = "application/json" + ip_address = Utils.ip_address(env) + protocol = Utils.protocol(env) + host = Utils.host(env) # You can modify this if you want to allow files smaller than 1MiB. # This is generally a good way to check the filesize but there is a better way to do it # which is inspecting the file directly (If I'm not wrong). if CONFIG.size_limit > 0 if env.request.headers["Content-Length"].to_i > 1048576*CONFIG.size_limit - error413("File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB") + return error413("File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB") end end filename = "" @@ -19,19 +22,18 @@ module Handling original_filename = "" uploaded_at = "" checksum = "" - ip_address = "" delete_key = nil # TODO: Return the file that matches a checksum inside the database HTTP::FormData.parse(env.request) do |upload| if upload.filename.nil? || upload.filename.to_s.empty? LOGGER.debug "No file provided by the user" - error403("No file provided") + return error403("No file provided") end # TODO: upload.body is emptied when is copied or read # Utils.check_duplicate(upload.dup) extension = File.extname("#{upload.filename}") if CONFIG.blockedExtensions.includes?(extension.split(".")[1]) - error401("Extension '#{extension}' is not allowed") + return error401("Extension '#{extension}' is not allowed") end filename = Utils.generate_filename file_path = ::File.join ["#{CONFIG.files}", filename + extension] @@ -68,17 +70,23 @@ module Handling # Insert SQL data just before returning the upload information SQL.exec "INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil + SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip) VALUES ('#{ip_address}')" + SQL.exec "UPDATE #{CONFIG.ipTableName} SET count = count + 1 WHERE ip = ('#{ip_address}')" + rescue ex LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" - error500("An error ocurred when trying to insert the data into the DB") + return error500("An error ocurred when trying to insert the data into the DB") end - return json + json end # The most unoptimized and unstable feature lol # TODO: Support batch upload via JSON array def upload_url(env) env.response.content_type = "application/json" + ip_address = Utils.ip_address(env) + protocol = Utils.protocol(env) + host = Utils.host(env) files = env.params.json["files"].as((Array(JSON::Any))) successfull_files = [] of NamedTuple(filename: String, extension: String, original_filename: String, checksum: String, delete_key: String | Nil) failed_files = [] of String @@ -103,7 +111,7 @@ module Handling end rescue ex LOGGER.debug "Failed to download file '#{url}': #{ex.message}" - error403("Failed to download file '#{url}'") + return error403("Failed to download file '#{url}'") failed_files << url end end @@ -134,7 +142,7 @@ module Handling checksum: checksum} rescue ex LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" - error500("An error ocurred when trying to insert the data into the DB") + return error500("An error ocurred when trying to insert the data into the DB") end end json = JSON.build do |j| @@ -156,11 +164,13 @@ module Handling end end end - return json + json end def retrieve_file(env) begin + protocol = Utils.protocol(env) + host = Utils.host(env) fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail FROM #{CONFIG.dbTableName} WHERE filename = ?", @@ -192,7 +202,7 @@ module Handling send_file env, "#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:ext]}" rescue ex LOGGER.debug "File '#{env.params.url["filename"]}' does not exist: #{ex.message}" - error403("File '#{env.params.url["filename"]}' does not exist") + return error403("File '#{env.params.url["filename"]}' does not exist") end end @@ -201,7 +211,7 @@ module Handling send_file env, "#{CONFIG.thumbnails}/#{env.params.url["thumbnail"]}" rescue ex LOGGER.debug "Thumbnail '#{env.params.url["thumbnail"]}' does not exist: #{ex.message}" - error403("Thumbnail '#{env.params.url["thumbnail"]}' does not exist") + return error403("Thumbnail '#{env.params.url["thumbnail"]}' does not exist") end end @@ -223,7 +233,7 @@ module Handling end rescue ex LOGGER.error "Unknown error: #{ex.message}" - error500("Unknown error") + return error500("Unknown error") end json_data end @@ -246,18 +256,20 @@ module Handling # Delete entry from db SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE delete_key = ?", env.params.query["key"] LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" - msg("File '#{fileinfo[:filename]}' deleted successfully") + return msg("File '#{fileinfo[:filename]}' deleted successfully") rescue ex LOGGER.error("Unknown error: #{ex.message}") - error500("Unknown error") + return error500("Unknown error") end else LOGGER.debug "Key '#{env.params.query["key"]}' does not exist" - error401("Delete key '#{env.params.query["key"]}' does not exist. No files were deleted") + return error401("Delete key '#{env.params.query["key"]}' does not exist. No files were deleted") end end def sharex_config(env) + host = Utils.host(env) + protocol = Utils.protocol(env) env.response.content_type = "application/json" # So it's able to download the file instead of displaying it env.response.headers["Content-Disposition"] = "attachment; filename=\"#{host}.sxcu\"" diff --git a/src/routing.cr b/src/routing.cr index c7e3209..20f71a9 100644 --- a/src/routing.cr +++ b/src/routing.cr @@ -1,20 +1,11 @@ require "./http-errors" -macro ip_address - env.request.headers.try &.["X-Forwarded-For"]? || env.request.remote_address.to_s.split(":").first -end - -macro protocol - env.request.headers.try &.["X-Forwarded-Proto"]? || "http" -end - -macro host - env.request.headers.try &.["X-Forwarded-Host"]? || env.request.headers["Host"] -end - module Routing extend self @@exit_nodes = Array(String).new + # @@ip_address : String = "" + # @@protocol : String = "" + # @@host : String = "" if CONFIG.blockTorAddresses spawn do # Wait a little for Utils.retrieve_tor_exit_nodes to execute first @@ -29,15 +20,20 @@ module Routing end end before_post do |env| - if @@exit_nodes.includes?(ip_address) + if @@exit_nodes.includes?(Utils.ip_address(env)) halt env, status_code: 401, response: error401(CONFIG.torMessage) end - end + ip_count = SQL.query_one "SELECT count FROM #{CONFIG.ipTableName} WHERE ip = ?", Utils.ip_address(env), as: Int32 + if ip_count >= CONFIG.filesPerIP + halt env, status_code: 401, response: error401("Rate limited!") + end + end end def register_all get "/" do |env| - files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 + host = Utils.host(env) + files_hosted = SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.dbTableName}", as: Int32 render "src/views/index.ecr" end diff --git a/src/utils.cr b/src/utils.cr index 0d8358d..66587fc 100644 --- a/src/utils.cr +++ b/src/utils.cr @@ -2,11 +2,14 @@ module Utils extend self def create_db - if !SQL.query_one "SELECT EXISTS (SELECT name FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.dbTableName}');", as: Bool + if !SQL.query_one "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.dbTableName}') + AND EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.ipTableName}');", as: Bool LOGGER.info "Creating sqlite3 database at '#{CONFIG.db}'" begin SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.dbTableName} (original_filename text, filename text, extension text, uploaded_at text, checksum text, ip text, delete_key text, thumbnail text)" + SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.ipTableName} + (ip text UNIQUE, count integer DEFAULT 0)" rescue ex LOGGER.fatal "#{ex.message}" exit(1) @@ -206,4 +209,28 @@ module Utils def load_tor_exit_nodes exit_nodes = File.read_lines(CONFIG.torExitNodesFile) end + + def ip_address(env) : String + begin + return env.request.headers.try &.["X-Forwarded-For"] + rescue + return env.request.remote_address.to_s.split(":").first + end + end + + def protocol(env) : String + begin + return env.request.headers.try &.["X-Forwarded-Proto"] + rescue + return "http" + end + end + + def host(env) : String + begin + return env.request.headers.try &.["X-Forwarded-Host"] + rescue + return env.request.headers["Host"] + end + end end