diff --git a/README.md b/README.md index efbb088..1f826ef 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Already replaced lol. - File deletion link (not available in frontend for now) - Chatterino and ShareX support - Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, can be disabled.) -- Rate Limiting (WIP) -- [Small Admin API](./src/handling/admin.cr) that allows you to delete files. (Needs to be enabled in the configuration) +- Rate Limiting +- [Small Admin API](./src/handling/admin.cr) that allows you to delete files, reset rate limits and more (Needs to be enabled in the configuration) - Unix socket support if you don't want to deal with all the TCP overhead - Automatic protocol detection (HTTPS or HTTP) - Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. It will depend of your traffic. @@ -83,3 +83,4 @@ WantedBy=default.target - Dockerfile and Docker image (Crystal doesn't has dependency hell like other languages so is not really necessary to do, but useful for people that want instant deploy) - Custom file expiration using headers (Like rustypaste) - Small CLI to upload files (like `rpaste` from rustypaste) +- Add more endpoints to Admin API \ No newline at end of file diff --git a/config/config.yml b/config/config.yml index 672f074..c24bc36 100644 --- a/config/config.yml +++ b/config/config.yml @@ -3,7 +3,7 @@ thumbnails: "./thumbnails" generateThumbnails: false db: "./db.sqlite3" dbTableName: "files" -adminEnabled: false +adminEnabled: true adminApiKey: "asd" fileameLength: 3 # In MiB @@ -15,6 +15,10 @@ torExitNodesCheck: 3600 torExitNodesUrl: "https://www.dan.me.uk/torlist/?exit" torExitNodesFile: "./torexitnodes.txt" torMessage: "TOR IS BLOCKED!" +filesPerIP: 2 +ipTableName: "ips" +rateLimitPeriod: 20 +rateLimitMessage: "" # If you define the unix socket, it will only listen on the socket and not the port. #unix_socket: "/tmp/file-uploader.sock" # In days diff --git a/src/config.cr b/src/config.cr index 4d9ae49..c94643d 100644 --- a/src/config.cr +++ b/src/config.cr @@ -3,39 +3,72 @@ require "yaml" class Config include YAML::Serializable + # Where the uploaded files will be located property files : String = "./files" + # Where the thumbnails will be located when they are successfully generated property thumbnails : String = "./thumbnails" + # Generate thumbnails for OpenGraph compatible platforms like Chatterino + # Whatsapp, Facebook, Discord, etc. property generateThumbnails : Bool = false + # Where the SQLITE3 database will be located property db : String = "./db.sqlite3" + # Name of the table that will be used for file information property dbTableName : String = "files" + # Enable or disable the admin API property adminEnabled : Bool = false + # The API key for admin routes. It's passed as a "X-Api-Key" header to the + # request property adminApiKey : String? = "" # Not implemented property incrementalFileameLength : Bool = true + # Filename length property fileameLength : Int32 = 3 # In MiB property size_limit : Int16 = 512 + # TCP port property port : Int32 = 8080 + # A file path where do you want to place a unix socket (THIS WILL DISABLE ACCESS + # BY IP ADDRESS) property unix_socket : String? + # True if you want this program to block IP addresses coming from the Tor + # network property blockTorAddresses : Bool? = false + # How often (in seconds) should this program download the exit nodes list property torExitNodesCheck : Int32 = 3600 + # A URL with a list of exit nodes addresses # The list needs to contain a IP address per line property torExitNodesUrl : String = "https://www.dan.me.uk/torlist/?exit" + # Where the file of the exit nodes will be located, can be placed anywhere property torExitNodesFile : String = "./torexitnodes.txt" - property filesPerIP : Int32 = 32 - property ipTableName : String = "ips" - # How often is the file limit per IP reset? - property rateLimitPeriod : Int32 = 600 + # Message that will be displayed to the Tor user. + # It will be shown on the Frontend and shown in the error 401 when a user + # tries to upload a file using curl or any other tool property torMessage : String? = "Tor is blocked!" + # How many files an IP address can upload to the server + property filesPerIP : Int32 = 32 + # Name of the table that will be used for rate limit information + property ipTableName : String = "ips" + # How often is the file limit per IP reset? (in seconds) + property rateLimitPeriod : Int32 = 600 + # TODO: UNUSED CONSTANT + property rateLimitMessage : String = "" + # Delete the files after how many days? property deleteFilesAfter : Int32 = 7 # How often should the check of old files be performed? (in seconds) property deleteFilesCheck : Int32 = 1800 + # The lenght of the delete key property deleteKeyLength : Int32 = 4 - # Blocked extensions that are not allowed to be uploaded to the server property siteInfo : String = "xd" + # TODO: UNUSED CONSTANT property siteWarning : String? = "" + # Log level property log_level : LogLevel = LogLevel::Info + # Blocked extensions that are not allowed to be uploaded to the server property blockedExtensions : Array(String) = [] of String + # A list of OpenGraph user agents. If the request contains one of those User + # agents when trying to retrieve a file from the server; the server will + # reply with an HTML with OpenGraph tags, pointing to the media thumbnail + # (if it was generated successfully) and the name of the file as title property opengraphUseragents : Array(String) = [] of String # Since this program detects the Host header of the client it can be used # with multiple domains. You can display the domains in the frontend diff --git a/src/handling/admin.cr b/src/handling/admin.cr index e403873..0035f53 100644 --- a/src/handling/admin.cr +++ b/src/handling/admin.cr @@ -3,10 +3,8 @@ require "../http-errors" module Handling::Admin extend self + # /api/admin/delete def delete_file(env) - if env.request.headers.try &.["X-Api-Key"]? != CONFIG.adminApiKey || nil - error401 "Wrong API Key" - end files = env.params.json["files"].as((Array(JSON::Any))) successfull_files = [] of String failed_files = [] of String @@ -46,4 +44,34 @@ module Handling::Admin end end end + + # /api/admin/deleteiplimit + def delete_ip_limit(env) + ips = env.params.json["ips"].as((Array(JSON::Any))) + successfull_ips = [] of String + failed_ips = [] of String + ips.each do |ip| + ip = ip.to_s + begin + # Delete entry from db + SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", ip + LOGGER.debug "Rate limit for '#{ip}' was deleted" + successfull_ips << ip + rescue ex : DB::NoResultsError + LOGGER.error("Rate limit for '#{ip}' doesn't exist or is not registered in the database: #{ex.message}") + failed_ips << ip + rescue ex + LOGGER.error "Unknown error: #{ex.message}" + error500 "Unknown error: #{ex.message}" + end + end + json = JSON.build do |j| + j.object do + j.field "successfull", successfull_ips.size + j.field "failed", failed_ips.size + j.field "successfullUnbans", successfull_ips + j.field "failedUnbans", failed_ips + end + end + end end diff --git a/src/handling/handling.cr b/src/handling/handling.cr index d408f71..f5f3c21 100644 --- a/src/handling/handling.cr +++ b/src/handling/handling.cr @@ -22,7 +22,9 @@ module Handling original_filename = "" uploaded_at = "" checksum = "" - delete_key = nil + if CONFIG.deleteKeyLength > 0 + delete_key = Random.base58(CONFIG.deleteKeyLength) + end # TODO: Return the file that matches a checksum inside the database HTTP::FormData.parse(env.request) do |upload| if upload.filename.nil? || upload.filename.to_s.empty? @@ -46,21 +48,6 @@ module Handling end # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse # proxy configuration. - json = JSON.build do |j| - j.object do - j.field "link", "#{protocol}://#{host}/#{filename}" - j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}" - j.field "id", filename - j.field "ext", extension - j.field "name", original_filename - j.field "checksum", checksum - if CONFIG.deleteKeyLength > 0 - delete_key = Random.base58(CONFIG.deleteKeyLength) - j.field "deleteKey", delete_key - j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}" - end - end - end begin spawn { Utils.generate_thumbnail(filename, extension) } rescue ex @@ -70,13 +57,27 @@ module Handling # Insert SQL data just before returning the upload information SQL.exec "INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil - SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip) VALUES ('#{ip_address}')" + SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip, date) VALUES (?, ?)", ip_address, Time.utc.to_unix + # SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip) VALUES ('#{ip_address}')" SQL.exec "UPDATE #{CONFIG.ipTableName} SET count = count + 1 WHERE ip = ('#{ip_address}')" - rescue ex LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" return error500("An error ocurred when trying to insert the data into the DB") end + json = JSON.build do |j| + j.object do + j.field "link", "#{protocol}://#{host}/#{filename}" + j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}" + j.field "id", filename + j.field "ext", extension + j.field "name", original_filename + j.field "checksum", checksum + if CONFIG.deleteKeyLength > 0 + j.field "deleteKey", delete_key + j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}" + end + end + end json end diff --git a/src/routing.cr b/src/routing.cr index 20f71a9..732cd83 100644 --- a/src/routing.cr +++ b/src/routing.cr @@ -3,9 +3,6 @@ require "./http-errors" module Routing extend self @@exit_nodes = Array(String).new - # @@ip_address : String = "" - # @@protocol : String = "" - # @@host : String = "" if CONFIG.blockTorAddresses spawn do # Wait a little for Utils.retrieve_tor_exit_nodes to execute first @@ -19,15 +16,39 @@ module Routing sleep CONFIG.torExitNodesCheck + 5 end end - before_post do |env| - if @@exit_nodes.includes?(Utils.ip_address(env)) - halt env, status_code: 401, response: error401(CONFIG.torMessage) + end + + before_post do |env| + if env.request.headers.try &.["X-Api-Key"]? == CONFIG.adminApiKey + # Skips Tor and Rate limits if the API key matches + next + end + if CONFIG.blockTorAddresses && @@exit_nodes.includes?(Utils.ip_address(env)) + halt env, status_code: 401, response: error401(CONFIG.torMessage) + end + # There is a better way to do this + if env.request.resource == "/upload" + begin + ip_info = SQL.query_all("SELECT ip, count, date FROM #{CONFIG.ipTableName} WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32})[0] + time_since_first_upload = Time.utc.to_unix - ip_info[:date] + time_until_unban = ip_info[:date] - Time.utc.to_unix + CONFIG.rateLimitPeriod + if time_since_first_upload > CONFIG.rateLimitPeriod + SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", ip_info[:ip] + end + if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod + halt env, status_code: 401, response: error401("Rate limited! Try again in #{time_until_unban} seconds") + end + rescue ex + LOGGER.error "Error when trying to enforce rate limits: #{ex.message}" + next end - ip_count = SQL.query_one "SELECT count FROM #{CONFIG.ipTableName} WHERE ip = ?", Utils.ip_address(env), as: Int32 - if ip_count >= CONFIG.filesPerIP - halt env, status_code: 401, response: error401("Rate limited!") - end - end + end + end + + before_post "/api/admin" do |env| + if env.request.headers.try &.["X-Api-Key"]? != CONFIG.adminApiKey || nil + error401 "Wrong API Key" + end end def register_all @@ -70,9 +91,15 @@ module Routing def register_admin if CONFIG.adminEnabled + post "/api/admin/upload" do |env| + Handling::Admin.delete_ip_limit(env) + end post "/api/admin/delete" do |env| Handling::Admin.delete_file(env) end end + post "/api/admin/deleteiplimit" do |env| + Handling::Admin.delete_ip_limit(env) + end end end diff --git a/src/utils.cr b/src/utils.cr index 66587fc..7f22cf8 100644 --- a/src/utils.cr +++ b/src/utils.cr @@ -9,7 +9,7 @@ module Utils SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.dbTableName} (original_filename text, filename text, extension text, uploaded_at text, checksum text, ip text, delete_key text, thumbnail text)" SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.ipTableName} - (ip text UNIQUE, count integer DEFAULT 0)" + (ip text UNIQUE, count integer DEFAULT 0, date integer)" rescue ex LOGGER.fatal "#{ex.message}" exit(1)