0.8.7: WIP built-in IP address rate limit

README.md

@@ -10,6 +10,7 @@ Already replaced lol.
 - File deletion link (not available in frontend for now)
 - Chatterino and ShareX support
 - Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, can be disabled.)
+- Rate Limiting (WIP)
 - [Small Admin API](./src/handling/admin.cr) that allows you to delete files. (Needs to be enabled in the configuration)
 - Unix socket support if you don't want to deal with all the TCP overhead
 - Automatic protocol detection (HTTPS or HTTP)

shard.yml

@@ -1,5 +1,5 @@
 name: file-uploader
-version: 0.8.0
+version: 0.8.7
 
 authors:
   - Fijxu <fijxu@nadeko.net>

src/config.cr

@@ -22,6 +22,10 @@ class Config
   # The list needs to contain a IP address per line
   property torExitNodesUrl : String = "https://www.dan.me.uk/torlist/?exit"
   property torExitNodesFile : String = "./torexitnodes.txt"
+  property filesPerIP : Int32 = 32
+  property ipTableName : String = "ips"
+  # How often is the file limit per IP reset?
+  property rateLimitPeriod : Int32 = 600
   property torMessage : String? = "Tor is blocked!"
   property deleteFilesAfter : Int32 = 7
   # How often should the check of old files be performed? (in seconds)

src/handling/handling.cr

@@ -6,12 +6,15 @@ module Handling
 
   def upload(env)
     env.response.content_type = "application/json"
+    ip_address = Utils.ip_address(env)
+    protocol = Utils.protocol(env)
+    host = Utils.host(env)
     # You can modify this if you want to allow files smaller than 1MiB.
     # This is generally a good way to check the filesize but there is a better way to do it
     # which is inspecting the file directly (If I'm not wrong).
     if CONFIG.size_limit > 0
       if env.request.headers["Content-Length"].to_i > 1048576*CONFIG.size_limit
-        error413("File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB")
+        return error413("File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB")
       end
     end
     filename = ""
@@ -19,19 +22,18 @@ module Handling
     original_filename = ""
     uploaded_at = ""
     checksum = ""
-    ip_address = ""
     delete_key = nil
     # TODO: Return the file that matches a checksum inside the database
     HTTP::FormData.parse(env.request) do |upload|
       if upload.filename.nil? || upload.filename.to_s.empty?
         LOGGER.debug "No file provided by the user"
-        error403("No file provided")
+        return error403("No file provided")
       end
       # TODO: upload.body is emptied when is copied or read
       # Utils.check_duplicate(upload.dup)
       extension = File.extname("#{upload.filename}")
       if CONFIG.blockedExtensions.includes?(extension.split(".")[1])
-        error401("Extension '#{extension}' is not allowed")
+        return error401("Extension '#{extension}' is not allowed")
       end
       filename = Utils.generate_filename
       file_path = ::File.join ["#{CONFIG.files}", filename + extension]
@@ -68,17 +70,23 @@ module Handling
       # Insert SQL data just before returning the upload information
       SQL.exec "INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
         original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
+      SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip) VALUES ('#{ip_address}')"
+      SQL.exec "UPDATE #{CONFIG.ipTableName} SET count = count + 1 WHERE ip = ('#{ip_address}')"
+
     rescue ex
       LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
-      error500("An error ocurred when trying to insert the data into the DB")
+      return error500("An error ocurred when trying to insert the data into the DB")
     end
-    return json
+    json
   end
 
   # The most unoptimized and unstable feature lol
   # TODO: Support batch upload via JSON array
   def upload_url(env)
     env.response.content_type = "application/json"
+    ip_address = Utils.ip_address(env)
+    protocol = Utils.protocol(env)
+    host = Utils.host(env)
     files = env.params.json["files"].as((Array(JSON::Any)))
     successfull_files = [] of NamedTuple(filename: String, extension: String, original_filename: String, checksum: String, delete_key: String | Nil)
     failed_files = [] of String
@@ -103,7 +111,7 @@ module Handling
           end
         rescue ex
           LOGGER.debug "Failed to download file '#{url}': #{ex.message}"
-          error403("Failed to download file '#{url}'")
+          return error403("Failed to download file '#{url}'")
           failed_files << url
         end
       end
@@ -134,7 +142,7 @@ module Handling
                               checksum:          checksum}
       rescue ex
         LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
-        error500("An error ocurred when trying to insert the data into the DB")
+        return error500("An error ocurred when trying to insert the data into the DB")
       end
     end
     json = JSON.build do |j|
@@ -156,11 +164,13 @@ module Handling
         end
       end
     end
-    return json
+    json
   end
 
   def retrieve_file(env)
     begin
+      protocol = Utils.protocol(env)
+      host = Utils.host(env)
       fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail
       FROM #{CONFIG.dbTableName}
       WHERE filename = ?",
@@ -192,7 +202,7 @@ module Handling
       send_file env, "#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:ext]}"
     rescue ex
       LOGGER.debug "File '#{env.params.url["filename"]}' does not exist: #{ex.message}"
-      error403("File '#{env.params.url["filename"]}' does not exist")
+      return error403("File '#{env.params.url["filename"]}' does not exist")
     end
   end
 
@@ -201,7 +211,7 @@ module Handling
       send_file env, "#{CONFIG.thumbnails}/#{env.params.url["thumbnail"]}"
     rescue ex
       LOGGER.debug "Thumbnail '#{env.params.url["thumbnail"]}' does not exist: #{ex.message}"
-      error403("Thumbnail '#{env.params.url["thumbnail"]}' does not exist")
+      return error403("Thumbnail '#{env.params.url["thumbnail"]}' does not exist")
     end
   end
 
@@ -223,7 +233,7 @@ module Handling
       end
     rescue ex
       LOGGER.error "Unknown error: #{ex.message}"
-      error500("Unknown error")
+      return error500("Unknown error")
     end
     json_data
   end
@@ -246,18 +256,20 @@ module Handling
         # Delete entry from db
         SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE delete_key = ?", env.params.query["key"]
         LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}"
-        msg("File '#{fileinfo[:filename]}' deleted successfully")
+        return msg("File '#{fileinfo[:filename]}' deleted successfully")
       rescue ex
         LOGGER.error("Unknown error: #{ex.message}")
-        error500("Unknown error")
+        return error500("Unknown error")
       end
     else
       LOGGER.debug "Key '#{env.params.query["key"]}' does not exist"
-      error401("Delete key '#{env.params.query["key"]}' does not exist. No files were deleted")
+      return error401("Delete key '#{env.params.query["key"]}' does not exist. No files were deleted")
     end
   end
 
   def sharex_config(env)
+    host = Utils.host(env)
+    protocol = Utils.protocol(env)
     env.response.content_type = "application/json"
     # So it's able to download the file instead of displaying it
     env.response.headers["Content-Disposition"] = "attachment; filename=\"#{host}.sxcu\""

src/routing.cr

@@ -1,20 +1,11 @@
 require "./http-errors"
 
-macro ip_address
-  env.request.headers.try &.["X-Forwarded-For"]? || env.request.remote_address.to_s.split(":").first
-end
-
-macro protocol
-  env.request.headers.try &.["X-Forwarded-Proto"]? || "http"
-end
-
-macro host
-  env.request.headers.try &.["X-Forwarded-Host"]? || env.request.headers["Host"]
-end
-
 module Routing
   extend self
   @@exit_nodes = Array(String).new
+  #   @@ip_address : String = ""
+  #   @@protocol : String = ""
+  #   @@host : String = ""
   if CONFIG.blockTorAddresses
     spawn do
       # Wait a little for Utils.retrieve_tor_exit_nodes to execute first
@@ -29,15 +20,20 @@ module Routing
       end
     end
     before_post do |env|
-      if @@exit_nodes.includes?(ip_address)
+      if @@exit_nodes.includes?(Utils.ip_address(env))
         halt env, status_code: 401, response: error401(CONFIG.torMessage)
       end
+      ip_count = SQL.query_one "SELECT count FROM #{CONFIG.ipTableName} WHERE ip = ?", Utils.ip_address(env), as: Int32
+	  if ip_count >= CONFIG.filesPerIP
+        halt env, status_code: 401, response: error401("Rate limited!")
+	  end
 	end
   end
 
   def register_all
     get "/" do |env|
-      files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32
+      host = Utils.host(env)
+      files_hosted = SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.dbTableName}", as: Int32
       render "src/views/index.ecr"
     end
 

src/utils.cr

@@ -2,11 +2,14 @@ module Utils
   extend self
 
   def create_db
-    if !SQL.query_one "SELECT EXISTS (SELECT name FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.dbTableName}');", as: Bool
+    if !SQL.query_one "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.dbTableName}')
+		AND EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.ipTableName}');", as: Bool
       LOGGER.info "Creating sqlite3 database at '#{CONFIG.db}'"
       begin
         SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.dbTableName}
 		(original_filename text, filename text, extension text, uploaded_at text, checksum text, ip text, delete_key text, thumbnail text)"
+        SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.ipTableName}
+		(ip text UNIQUE, count integer DEFAULT 0)"
       rescue ex
         LOGGER.fatal "#{ex.message}"
         exit(1)
@@ -206,4 +209,28 @@ module Utils
   def load_tor_exit_nodes
     exit_nodes = File.read_lines(CONFIG.torExitNodesFile)
   end
+
+  def ip_address(env) : String
+    begin
+      return env.request.headers.try &.["X-Forwarded-For"]
+    rescue
+      return env.request.remote_address.to_s.split(":").first
+    end
+  end
+
+  def protocol(env) : String
+    begin
+      return env.request.headers.try &.["X-Forwarded-Proto"]
+    rescue
+      return "http"
+    end
+  end
+
+  def host(env) : String
+	begin
+      return env.request.headers.try &.["X-Forwarded-Host"]
+    rescue
+      return env.request.headers["Host"]
+    end
+  end
 end