0.8.8: Rate limit completed, fix delete_key

README.md

@@ -10,8 +10,8 @@ Already replaced lol.
 - File deletion link (not available in frontend for now)
 - Chatterino and ShareX support
 - Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, can be disabled.)
-- Rate Limiting (WIP)
+- Rate Limiting
-- [Small Admin API](./src/handling/admin.cr) that allows you to delete files. (Needs to be enabled in the configuration)
+- [Small Admin API](./src/handling/admin.cr) that allows you to delete files, reset rate limits and more (Needs to be enabled in the configuration)
 - Unix socket support if you don't want to deal with all the TCP overhead
 - Automatic protocol detection (HTTPS or HTTP)
 - Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. It will depend of your traffic.
@@ -83,3 +83,4 @@ WantedBy=default.target
 - Dockerfile and Docker image (Crystal doesn't has dependency hell like other languages so is not really necessary to do, but useful for people that want instant deploy)
 - Custom file expiration using headers (Like rustypaste)
 - Small CLI to upload files (like `rpaste` from rustypaste)
+- Add more endpoints to Admin API

config/config.yml

@@ -3,7 +3,7 @@ thumbnails: "./thumbnails"
 generateThumbnails: false
 db: "./db.sqlite3"
 dbTableName: "files"
-adminEnabled: false
+adminEnabled: true
 adminApiKey: "asd"
 fileameLength: 3
 # In MiB
@@ -15,6 +15,10 @@ torExitNodesCheck: 3600
 torExitNodesUrl: "https://www.dan.me.uk/torlist/?exit"
 torExitNodesFile: "./torexitnodes.txt"
 torMessage: "TOR IS BLOCKED!"
+filesPerIP: 2
+ipTableName: "ips"
+rateLimitPeriod: 20
+rateLimitMessage: ""
 # If you define the unix socket, it will only listen on the socket and not the port.
 #unix_socket: "/tmp/file-uploader.sock"
 # In days

src/config.cr

@@ -3,39 +3,72 @@ require "yaml"
 class Config
   include YAML::Serializable
 
+  # Where the uploaded files will be located
   property files : String = "./files"
+  # Where the thumbnails will be located when they are successfully generated
   property thumbnails : String = "./thumbnails"
+  # Generate thumbnails for OpenGraph compatible platforms like Chatterino
+  # Whatsapp, Facebook, Discord, etc.
   property generateThumbnails : Bool = false
+  # Where the SQLITE3 database will be located
   property db : String = "./db.sqlite3"
+  # Name of the table that will be used for file information
   property dbTableName : String = "files"
+  # Enable or disable the admin API
   property adminEnabled : Bool = false
+  # The API key for admin routes. It's passed as a "X-Api-Key" header to the
+  # request
   property adminApiKey : String? = ""
   # Not implemented
   property incrementalFileameLength : Bool = true
+  # Filename length
   property fileameLength : Int32 = 3
   # In MiB
   property size_limit : Int16 = 512
+  # TCP port
   property port : Int32 = 8080
+  # A file path where do you want to place a unix socket (THIS WILL DISABLE ACCESS
+  # BY IP ADDRESS)
   property unix_socket : String?
+  # True if you want this program to block IP addresses coming from the Tor
+  # network
   property blockTorAddresses : Bool? = false
+  # How often (in seconds) should this program download the exit nodes list
   property torExitNodesCheck : Int32 = 3600
+  # A URL with a list of exit nodes addresses
   # The list needs to contain a IP address per line
   property torExitNodesUrl : String = "https://www.dan.me.uk/torlist/?exit"
+  # Where the file of the exit nodes will be located, can be placed anywhere
   property torExitNodesFile : String = "./torexitnodes.txt"
-  property filesPerIP : Int32 = 32
+  # Message that will be displayed to the Tor user.
-  property ipTableName : String = "ips"
+  # It will be shown on the Frontend and shown in the error 401 when a user
-  # How often is the file limit per IP reset?
+  # tries to upload a file using curl or any other tool
-  property rateLimitPeriod : Int32 = 600
   property torMessage : String? = "Tor is blocked!"
+  # How many files an IP address can upload to the server
+  property filesPerIP : Int32 = 32
+  # Name of the table that will be used for rate limit information
+  property ipTableName : String = "ips"
+  # How often is the file limit per IP reset? (in seconds)
+  property rateLimitPeriod : Int32 = 600
+  # TODO: UNUSED CONSTANT
+  property rateLimitMessage : String = ""
+  # Delete the files after how many days?
   property deleteFilesAfter : Int32 = 7
   # How often should the check of old files be performed? (in seconds)
   property deleteFilesCheck : Int32 = 1800
+  # The lenght of the delete key
   property deleteKeyLength : Int32 = 4
-  # Blocked extensions that are not allowed to be uploaded to the server
   property siteInfo : String = "xd"
+  # TODO: UNUSED CONSTANT
   property siteWarning : String? = ""
+  # Log level
   property log_level : LogLevel = LogLevel::Info
+  # Blocked extensions that are not allowed to be uploaded to the server
   property blockedExtensions : Array(String) = [] of String
+  # A list of OpenGraph user agents. If the request contains one of those User
+  # agents when trying to retrieve a file from the server; the server will
+  # reply with an HTML with OpenGraph tags, pointing to the media thumbnail
+  # (if it was generated successfully) and the name of the file as title
   property opengraphUseragents : Array(String) = [] of String
   # Since this program detects the Host header of the client it can be used
   # with multiple domains. You can display the domains in the frontend

src/handling/admin.cr

@@ -3,10 +3,8 @@ require "../http-errors"
 module Handling::Admin
   extend self
 
+  # /api/admin/delete
   def delete_file(env)
-    if env.request.headers.try &.["X-Api-Key"]? != CONFIG.adminApiKey || nil
-      error401 "Wrong API Key"
-    end
     files = env.params.json["files"].as((Array(JSON::Any)))
     successfull_files = [] of String
     failed_files = [] of String
@@ -46,4 +44,34 @@ module Handling::Admin
       end
     end
   end
+
+  # /api/admin/deleteiplimit
+  def delete_ip_limit(env)
+    ips = env.params.json["ips"].as((Array(JSON::Any)))
+    successfull_ips = [] of String
+    failed_ips = [] of String
+    ips.each do |ip|
+      ip = ip.to_s
+      begin
+        # Delete entry from db
+        SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", ip
+        LOGGER.debug "Rate limit for '#{ip}' was deleted"
+        successfull_ips << ip 
+      rescue ex : DB::NoResultsError
+        LOGGER.error("Rate limit for '#{ip}' doesn't exist or is not registered in the database: #{ex.message}")
+        failed_ips << ip
+      rescue ex
+        LOGGER.error "Unknown error: #{ex.message}"
+        error500 "Unknown error: #{ex.message}"
+      end
+    end
+    json = JSON.build do |j|
+      j.object do
+        j.field "successfull", successfull_ips.size
+        j.field "failed", failed_ips.size
+        j.field "successfullUnbans", successfull_ips
+        j.field "failedUnbans", failed_ips
+      end
+    end
+  end
 end

src/handling/handling.cr

@@ -22,7 +22,9 @@ module Handling
     original_filename = ""
     uploaded_at = ""
     checksum = ""
-    delete_key = nil
+    if CONFIG.deleteKeyLength > 0
+      delete_key = Random.base58(CONFIG.deleteKeyLength)
+    end
     # TODO: Return the file that matches a checksum inside the database
     HTTP::FormData.parse(env.request) do |upload|
       if upload.filename.nil? || upload.filename.to_s.empty?
@@ -46,21 +48,6 @@ module Handling
     end
     # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
     # proxy configuration.
-    json = JSON.build do |j|
-      j.object do
-        j.field "link", "#{protocol}://#{host}/#{filename}"
-        j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
-        j.field "id", filename
-        j.field "ext", extension
-        j.field "name", original_filename
-        j.field "checksum", checksum
-        if CONFIG.deleteKeyLength > 0
-          delete_key = Random.base58(CONFIG.deleteKeyLength)
-          j.field "deleteKey", delete_key
-          j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
-        end
-      end
-    end
     begin
       spawn { Utils.generate_thumbnail(filename, extension) }
     rescue ex
@@ -70,13 +57,27 @@ module Handling
       # Insert SQL data just before returning the upload information
       SQL.exec "INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
         original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
-      SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip) VALUES ('#{ip_address}')"
+      SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip, date) VALUES (?, ?)", ip_address, Time.utc.to_unix
+      # SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip) VALUES ('#{ip_address}')"
       SQL.exec "UPDATE #{CONFIG.ipTableName} SET count = count + 1 WHERE ip = ('#{ip_address}')"
-
     rescue ex
       LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
       return error500("An error ocurred when trying to insert the data into the DB")
     end
+    json = JSON.build do |j|
+      j.object do
+        j.field "link", "#{protocol}://#{host}/#{filename}"
+        j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
+        j.field "id", filename
+        j.field "ext", extension
+        j.field "name", original_filename
+        j.field "checksum", checksum
+        if CONFIG.deleteKeyLength > 0
+          j.field "deleteKey", delete_key
+          j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
+        end
+      end
+    end
     json
   end
 

src/routing.cr

@@ -3,9 +3,6 @@ require "./http-errors"
 module Routing
   extend self
   @@exit_nodes = Array(String).new
-  #   @@ip_address : String = ""
-  #   @@protocol : String = ""
-  #   @@host : String = ""
   if CONFIG.blockTorAddresses
     spawn do
       # Wait a little for Utils.retrieve_tor_exit_nodes to execute first
@@ -19,15 +16,39 @@ module Routing
         sleep CONFIG.torExitNodesCheck + 5
       end
     end
-    before_post do |env|
+  end
-      if @@exit_nodes.includes?(Utils.ip_address(env))
+
-        halt env, status_code: 401, response: error401(CONFIG.torMessage)
+  before_post do |env|
+    if env.request.headers.try &.["X-Api-Key"]? == CONFIG.adminApiKey
+      # Skips Tor and Rate limits if the API key matches
+      next
+    end
+    if CONFIG.blockTorAddresses && @@exit_nodes.includes?(Utils.ip_address(env))
+      halt env, status_code: 401, response: error401(CONFIG.torMessage)
+    end
+    # There is a better way to do this
+    if env.request.resource == "/upload"
+      begin
+        ip_info = SQL.query_all("SELECT ip, count, date FROM #{CONFIG.ipTableName} WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32})[0]
+        time_since_first_upload = Time.utc.to_unix - ip_info[:date]
+        time_until_unban = ip_info[:date] - Time.utc.to_unix + CONFIG.rateLimitPeriod
+        if time_since_first_upload > CONFIG.rateLimitPeriod
+          SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", ip_info[:ip]
+        end
+        if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod
+          halt env, status_code: 401, response: error401("Rate limited! Try again in #{time_until_unban} seconds")
+        end
+      rescue ex
+        LOGGER.error "Error when trying to enforce rate limits: #{ex.message}"
+        next
       end
-      ip_count = SQL.query_one "SELECT count FROM #{CONFIG.ipTableName} WHERE ip = ?", Utils.ip_address(env), as: Int32
+    end
-	  if ip_count >= CONFIG.filesPerIP
+  end
-        halt env, status_code: 401, response: error401("Rate limited!")
+
-	  end
+  before_post "/api/admin" do |env|
-	end
+    if env.request.headers.try &.["X-Api-Key"]? != CONFIG.adminApiKey || nil
+      error401 "Wrong API Key"
+    end
   end
 
   def register_all
@@ -70,9 +91,15 @@ module Routing
 
   def register_admin
     if CONFIG.adminEnabled
+      post "/api/admin/upload" do |env|
+        Handling::Admin.delete_ip_limit(env)
+      end
       post "/api/admin/delete" do |env|
         Handling::Admin.delete_file(env)
       end
     end
+    post "/api/admin/deleteiplimit" do |env|
+      Handling::Admin.delete_ip_limit(env)
+    end
   end
 end

src/utils.cr

@@ -9,7 +9,7 @@ module Utils
         SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.dbTableName}
 		(original_filename text, filename text, extension text, uploaded_at text, checksum text, ip text, delete_key text, thumbnail text)"
         SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.ipTableName}
-		(ip text UNIQUE, count integer DEFAULT 0)"
+		(ip text UNIQUE, count integer DEFAULT 0, date integer)"
       rescue ex
         LOGGER.fatal "#{ex.message}"
         exit(1)