Fijxu/file-uploader-crystal
1
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages 1 Wiki Activity Actions

0.8.8: Rate limit completed, fix delete_key

Browse source
This commit is contained in:
Fijxu 2024-08-18 21:47:57 -04:00
parent 80e230a0a9
commit cea1982523
Signed by: Fijxu
GPG key ID: 32C1DDF333EDA6A4
7 changed files with 135 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
README.md
View file

@ -10,8 +10,8 @@ Already replaced lol.
- File deletion link (not available in frontend for now)
- Chatterino and ShareX support
- Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, can be disabled.)
- Rate Limiting (WIP)
- [Small Admin API](./src/handling/admin.cr) that allows you to delete files. (Needs to be enabled in the configuration)
- Rate Limiting
- [Small Admin API](./src/handling/admin.cr) that allows you to delete files, reset rate limits and more (Needs to be enabled in the configuration)
- Unix socket support if you don't want to deal with all the TCP overhead
- Automatic protocol detection (HTTPS or HTTP)
- Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. It will depend of your traffic.
@ -83,3 +83,4 @@ WantedBy=default.target
- Dockerfile and Docker image (Crystal doesn't has dependency hell like other languages so is not really necessary to do, but useful for people that want instant deploy)
- Custom file expiration using headers (Like rustypaste)
- Small CLI to upload files (like `rpaste` from rustypaste)
- Add more endpoints to Admin API

6
config/config.yml
View file

@ -3,7 +3,7 @@ thumbnails: "./thumbnails"
generateThumbnails: false
db: "./db.sqlite3"
dbTableName: "files"
adminEnabled: false
adminEnabled: true
adminApiKey: "asd"
fileameLength: 3
# In MiB
@ -15,6 +15,10 @@ torExitNodesCheck: 3600
torExitNodesUrl: "https://www.dan.me.uk/torlist/?exit"
torExitNodesFile: "./torexitnodes.txt"
torMessage: "TOR IS BLOCKED!"
filesPerIP: 2
ipTableName: "ips"
rateLimitPeriod: 20
rateLimitMessage: ""
# If you define the unix socket, it will only listen on the socket and not the port.
#unix_socket: "/tmp/file-uploader.sock"
# In days

43
src/config.cr
View file

@ -3,39 +3,72 @@ require "yaml"
class Config
include YAML::Serializable
# Where the uploaded files will be located
property files : String = "./files"
# Where the thumbnails will be located when they are successfully generated
property thumbnails : String = "./thumbnails"
# Generate thumbnails for OpenGraph compatible platforms like Chatterino
# Whatsapp, Facebook, Discord, etc.
property generateThumbnails : Bool = false
# Where the SQLITE3 database will be located
property db : String = "./db.sqlite3"
# Name of the table that will be used for file information
property dbTableName : String = "files"
# Enable or disable the admin API
property adminEnabled : Bool = false
# The API key for admin routes. It's passed as a "X-Api-Key" header to the
# request
property adminApiKey : String? = ""
# Not implemented
property incrementalFileameLength : Bool = true
# Filename length
property fileameLength : Int32 = 3
# In MiB
property size_limit : Int16 = 512
# TCP port
property port : Int32 = 8080
# A file path where do you want to place a unix socket (THIS WILL DISABLE ACCESS
# BY IP ADDRESS)
property unix_socket : String?
# True if you want this program to block IP addresses coming from the Tor
# network
property blockTorAddresses : Bool? = false
# How often (in seconds) should this program download the exit nodes list
property torExitNodesCheck : Int32 = 3600
# A URL with a list of exit nodes addresses
# The list needs to contain a IP address per line
property torExitNodesUrl : String = "https://www.dan.me.uk/torlist/?exit"
# Where the file of the exit nodes will be located, can be placed anywhere
property torExitNodesFile : String = "./torexitnodes.txt"
property filesPerIP : Int32 = 32
property ipTableName : String = "ips"
# How often is the file limit per IP reset?
property rateLimitPeriod : Int32 = 600
# Message that will be displayed to the Tor user.
# It will be shown on the Frontend and shown in the error 401 when a user
# tries to upload a file using curl or any other tool
property torMessage : String? = "Tor is blocked!"
# How many files an IP address can upload to the server
property filesPerIP : Int32 = 32
# Name of the table that will be used for rate limit information
property ipTableName : String = "ips"
# How often is the file limit per IP reset? (in seconds)
property rateLimitPeriod : Int32 = 600
# TODO: UNUSED CONSTANT
property rateLimitMessage : String = ""
# Delete the files after how many days?
property deleteFilesAfter : Int32 = 7
# How often should the check of old files be performed? (in seconds)
property deleteFilesCheck : Int32 = 1800
# The lenght of the delete key
property deleteKeyLength : Int32 = 4
# Blocked extensions that are not allowed to be uploaded to the server
property siteInfo : String = "xd"
# TODO: UNUSED CONSTANT
property siteWarning : String? = ""
# Log level
property log_level : LogLevel = LogLevel::Info
# Blocked extensions that are not allowed to be uploaded to the server
property blockedExtensions : Array(String) = [] of String
# A list of OpenGraph user agents. If the request contains one of those User
# agents when trying to retrieve a file from the server; the server will
# reply with an HTML with OpenGraph tags, pointing to the media thumbnail
# (if it was generated successfully) and the name of the file as title
property opengraphUseragents : Array(String) = [] of String
# Since this program detects the Host header of the client it can be used
# with multiple domains. You can display the domains in the frontend

34
src/handling/admin.cr
View file

@ -3,10 +3,8 @@ require "../http-errors"
module Handling::Admin
extend self
# /api/admin/delete
def delete_file(env)
if env.request.headers.try &.["X-Api-Key"]? != CONFIG.adminApiKey || nil
error401 "Wrong API Key"
end
files = env.params.json["files"].as((Array(JSON::Any)))
successfull_files = [] of String
failed_files = [] of String
@ -46,4 +44,34 @@ module Handling::Admin
end
end
end
# /api/admin/deleteiplimit
def delete_ip_limit(env)
ips = env.params.json["ips"].as((Array(JSON::Any)))
successfull_ips = [] of String
failed_ips = [] of String
ips.each do |ip|
ip = ip.to_s
begin
# Delete entry from db
SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", ip
LOGGER.debug "Rate limit for '#{ip}' was deleted"
successfull_ips << ip
rescue ex : DB::NoResultsError
LOGGER.error("Rate limit for '#{ip}' doesn't exist or is not registered in the database: #{ex.message}")
failed_ips << ip
rescue ex
LOGGER.error "Unknown error: #{ex.message}"
error500 "Unknown error: #{ex.message}"
end
end
json = JSON.build do |j|
j.object do
j.field "successfull", successfull_ips.size
j.field "failed", failed_ips.size
j.field "successfullUnbans", successfull_ips
j.field "failedUnbans", failed_ips
end
end
end
end

37
src/handling/handling.cr
View file

@ -22,7 +22,9 @@ module Handling
original_filename = ""
uploaded_at = ""
checksum = ""
delete_key = nil
if CONFIG.deleteKeyLength > 0
delete_key = Random.base58(CONFIG.deleteKeyLength)
end
# TODO: Return the file that matches a checksum inside the database
HTTP::FormData.parse(env.request) do |upload|
if upload.filename.nil? || upload.filename.to_s.empty?
@ -46,21 +48,6 @@ module Handling
end
# X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
# proxy configuration.
json = JSON.build do |j|
j.object do
j.field "link", "#{protocol}://#{host}/#{filename}"
j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
j.field "id", filename
j.field "ext", extension
j.field "name", original_filename
j.field "checksum", checksum
if CONFIG.deleteKeyLength > 0
delete_key = Random.base58(CONFIG.deleteKeyLength)
j.field "deleteKey", delete_key
j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
end
end
end
begin
spawn { Utils.generate_thumbnail(filename, extension) }
rescue ex
@ -70,13 +57,27 @@ module Handling
# Insert SQL data just before returning the upload information
SQL.exec "INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip) VALUES ('#{ip_address}')"
SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip, date) VALUES (?, ?)", ip_address, Time.utc.to_unix
# SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip) VALUES ('#{ip_address}')"
SQL.exec "UPDATE #{CONFIG.ipTableName} SET count = count + 1 WHERE ip = ('#{ip_address}')"
rescue ex
LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
return error500("An error ocurred when trying to insert the data into the DB")
end
json = JSON.build do |j|
j.object do
j.field "link", "#{protocol}://#{host}/#{filename}"
j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
j.field "id", filename
j.field "ext", extension
j.field "name", original_filename
j.field "checksum", checksum
if CONFIG.deleteKeyLength > 0
j.field "deleteKey", delete_key
j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
end
end
end
json
end

49
src/routing.cr
View file

@ -3,9 +3,6 @@ require "./http-errors"
module Routing
extend self
@@exit_nodes = Array(String).new
# @@ip_address : String = ""
# @@protocol : String = ""
# @@host : String = ""
if CONFIG.blockTorAddresses
spawn do
# Wait a little for Utils.retrieve_tor_exit_nodes to execute first
@ -19,15 +16,39 @@ module Routing
sleep CONFIG.torExitNodesCheck + 5
end
end
before_post do |env|
if @@exit_nodes.includes?(Utils.ip_address(env))
halt env, status_code: 401, response: error401(CONFIG.torMessage)
end
before_post do |env|
if env.request.headers.try &.["X-Api-Key"]? == CONFIG.adminApiKey
# Skips Tor and Rate limits if the API key matches
next
end
if CONFIG.blockTorAddresses && @@exit_nodes.includes?(Utils.ip_address(env))
halt env, status_code: 401, response: error401(CONFIG.torMessage)
end
# There is a better way to do this
if env.request.resource == "/upload"
begin
ip_info = SQL.query_all("SELECT ip, count, date FROM #{CONFIG.ipTableName} WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32})[0]
time_since_first_upload = Time.utc.to_unix - ip_info[:date]
time_until_unban = ip_info[:date] - Time.utc.to_unix + CONFIG.rateLimitPeriod
if time_since_first_upload > CONFIG.rateLimitPeriod
SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", ip_info[:ip]
end
if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod
halt env, status_code: 401, response: error401("Rate limited! Try again in #{time_until_unban} seconds")
end
rescue ex
LOGGER.error "Error when trying to enforce rate limits: #{ex.message}"
next
end
ip_count = SQL.query_one "SELECT count FROM #{CONFIG.ipTableName} WHERE ip = ?", Utils.ip_address(env), as: Int32
if ip_count >= CONFIG.filesPerIP
halt env, status_code: 401, response: error401("Rate limited!")
end
end
end
end
before_post "/api/admin" do |env|
if env.request.headers.try &.["X-Api-Key"]? != CONFIG.adminApiKey || nil
error401 "Wrong API Key"
end
end
def register_all
@ -70,9 +91,15 @@ module Routing
def register_admin
if CONFIG.adminEnabled
post "/api/admin/upload" do |env|
Handling::Admin.delete_ip_limit(env)
end
post "/api/admin/delete" do |env|
Handling::Admin.delete_file(env)
end
end
post "/api/admin/deleteiplimit" do |env|
Handling::Admin.delete_ip_limit(env)
end
end
end

2
src/utils.cr
View file

@ -9,7 +9,7 @@ module Utils
SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.dbTableName}
(original_filename text, filename text, extension text, uploaded_at text, checksum text, ip text, delete_key text, thumbnail text)"
SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.ipTableName}
(ip text UNIQUE, count integer DEFAULT 0)"
(ip text UNIQUE, count integer DEFAULT 0, date integer)"
rescue ex
LOGGER.fatal "#{ex.message}"
exit(1)