From b19c423648ac5540bde93e54e91531920fd10f5e Mon Sep 17 00:00:00 2001 From: Fijxu Date: Tue, 22 Apr 2025 18:59:54 -0400 Subject: [PATCH] 0.9.6: Re-enable IP rate limits, add ips database logic and idk what more --- .gitignore | 3 +- config/config.example.yml | 41 ++++---- src/config.cr | 23 ++--- src/database/ip.cr | 47 ++++++++- src/macros.cr | 6 +- src/routes/delete.cr | 2 +- src/routes/retrieve.cr | 13 +-- src/routes/upload.cr | 200 +++----------------------------------- src/routing.cr | 26 ++--- src/types/ip.cr | 12 +-- src/types/ufile.cr | 18 ++-- src/utils/hashing.cr | 4 +- src/utils/utils.cr | 70 +++++++++++-- 13 files changed, 192 insertions(+), 273 deletions(-) diff --git a/.gitignore b/.gitignore index 94285f5..979d087 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ data torexitnodes.txt files thumbnails -db.sqlite3 \ No newline at end of file +db.sqlite3 +config/config.yml \ No newline at end of file diff --git a/config/config.example.yml b/config/config.example.yml index d3e921d..443b421 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -1,39 +1,37 @@ colorize_logs: true +log_level: "debug" + +# File paths files: "./files" thumbnails: "./thumbnails" -generate_thumbnails: true -db: "./db/db.sqlite3" -adminEnabled: true -adminApiKey: "asd" -filename_length: 3 -# In MiB -size_limit: 512 -port: 8080 +db: "./db.sqlite3" + +# Tor blockTorAddresses: true -# Every hour torExitNodesCheck: 1600 torExitNodesUrl: "https://check.torproject.org/exit-addresses" -torExitNodesFile: "./torexitnodes.txt" torMessage: "TOR IS BLOCKED!" -# Set this to 0 to disable rate limiting + +generate_thumbnails: true +adminEnabled: true +adminApiKey: "asd" +size_limit: 512 +enable_checksums: false +port: 8080 filesPerIP: 2 rateLimitPeriod: 20 rateLimitMessage: "" -# If you define the unix socket, it will only listen on the socket and not the port. -#unix_socket: "/tmp/file-uploader.sock" -# In days +filename_length: 3 deleteFilesAfter: 7 -# In seconds -deleteFilesCheck: 1600 +deleteFilesCheck: 1800 deleteKeyLength: 4 + siteInfo: "Whatever you want to put here" siteWarning: "WARNING!" -log_level: "debug" - + blockedExtensions: - "exe" -# List of useragents that use OpenGraph to gather file information opengraphUseragents: - "chatterino-api-cache/" - "FFZBot/" @@ -41,6 +39,5 @@ opengraphUseragents: - "Synapse/" - "Mastodon/" -# You can leave it empty, or add your own domains. -alternative_domains: - - "example.com" +# alternative_domains: +# - "example.com" diff --git a/src/config.cr b/src/config.cr index eccdb4e..9446f68 100644 --- a/src/config.cr +++ b/src/config.cr @@ -26,7 +26,7 @@ class Config property adminEnabled : Bool = false # The API key for admin routes. It's passed as a "X-Api-Key" header to the # request - property adminApiKey : String? = "" + property adminApiKey : String? = nil # Not implemented property incrementalfilename_length : Bool = true @@ -34,6 +34,7 @@ class Config property filename_length : Int32 = 3 # In MiB property size_limit : Int16 = 512 + property enable_checksums : Bool = true # A file path where do you want to place a unix socket (THIS WILL DISABLE ACCESS # BY IP ADDRESS) @@ -41,7 +42,7 @@ class Config # True if you want this program to block IP addresses coming from the Tor # network - property blockTorAddresses : Bool? = false + property blockTorAddresses : Bool = false # How often (in seconds) should this program download the exit nodes list property torExitNodesCheck : Int32 = 3600 # Only https://check.torproject.org/exit-addresses is supported @@ -51,7 +52,8 @@ class Config # tries to upload a file using curl or any other tool property torMessage : String? = "Tor is blocked!" - # How many files an IP address can upload to the server + # How many files an IP address can upload to the server. Setting this to 0 + # disables rate limits property filesPerIP : Int32 = 32 # How often is the file limit per IP reset? (in seconds) property rateLimitPeriod : Int32 = 600 @@ -83,14 +85,6 @@ class Config # and in `/api/stats` property alternative_domains : Array(String) = [] of String - def self.load - config_file = "config/config.yml" - config_yaml = File.read(config_file) - config = Config.from_yaml(config_yaml) - check_config(config) - config - end - def self.check_config(config : Config) if config.filename_length <= 0 puts "Config: filename_length cannot be less or equal to 0" @@ -104,4 +98,11 @@ class Config config.thumbnails = config.thumbnails.chomp('/') end end + + def self.load(config_file : String = "config/config.yml") + config_yaml = File.read(config_file) + config = Config.from_yaml(config_yaml) + check_config(config) + config + end end diff --git a/src/database/ip.cr b/src/database/ip.cr index c818185..10f926c 100644 --- a/src/database/ip.cr +++ b/src/database/ip.cr @@ -1,16 +1,55 @@ module Database::IP + extend self + # ------------------- # Insert / Delete # ------------------- - def insert(ip : IP) : Nil + def insert(ip : UIP) : DB::ExecResult request = <<-SQL INSERT OR IGNORE - INTO ips (ip, date) - VALUES ($1, $2) - ON CONFLICT DO NOTHING + INTO ips + VALUES ($1, $2, $3) SQL SQL.exec(request, *ip.to_tuple) end + + def delete(ip : String) : Nil + request = <<-SQL + DELETE + FROM ips + WHERE ip = ? + SQL + + SQL.exec(request, ip) + end + + # ------------------- + # Select + # ------------------- + + def select(ip : String) : UIP? + request = <<-SQL + SELECT * + FROM ips + WHERE ip = ? + SQL + + SQL.query_one?(request, ip, as: UIP) + end + + # ------------------- + # Update + # ------------------- + + def increase_count(ip : UIP) : Nil + request = <<-SQL + UPDATE ips + SET count = count + 1 + WHERE ip = $1 + SQL + + SQL.exec(request, ip.ip) + end end diff --git a/src/macros.cr b/src/macros.cr index 972473f..79b00a3 100644 --- a/src/macros.cr +++ b/src/macros.cr @@ -13,14 +13,14 @@ end module Headers macro host - env.request.headers["X-Forwarded-Host"]? + env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]? end macro scheme - env.request.headers["X-Forwarded-Proto"]? + env.request.headers["X-Forwarded-Proto"]? || "http" end macro ip_addr - env.request.headers["X-Real-IP"]? + env.request.headers["X-Real-IP"]? || env.request.remote_address.as?(Socket::IPAddress).try &.address end end diff --git a/src/routes/delete.cr b/src/routes/delete.cr index 2467174..267cdf7 100644 --- a/src/routes/delete.cr +++ b/src/routes/delete.cr @@ -5,7 +5,7 @@ module Routes::Deletion key = env.params.query["key"]? if !key || key.empty? - ee 400, "No delete key suplied" + ee 400, "No delete key supplied" end file = Database::Files.select_with_key(key) diff --git a/src/routes/retrieve.cr b/src/routes/retrieve.cr index 144c8a4..a9cd85c 100644 --- a/src/routes/retrieve.cr +++ b/src/routes/retrieve.cr @@ -4,7 +4,7 @@ module Routes::Retrieve def retrieve_file(env) host = env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]? scheme = env.request.headers["X-Forwarded-Proto"]? || "http" - ip_addr = env.request.headers["X-Real-IP"]? || env.request.remote_address + ip_addr = env.request.headers["X-Real-IP"]? || env.request.remote_address.as?(Socket::IPAddress).try &.address filename = env.params.url["filename"].split(".").first begin @@ -23,18 +23,16 @@ module Routes::Retrieve CONFIG.opengraphUseragents.each do |useragent| env.response.content_type = "text/html" - if env.request.headers.["User-Agent"]?.try &.includes?(useragent) + if env.request.headers["User-Agent"]?.try &.includes?(useragent) return %( - + - #{if file.thumbnail - %() - end} - + #{%() if file.thumbnail} + ) end @@ -44,7 +42,6 @@ module Routes::Retrieve def retrieve_thumbnail(env) thumbnail = env.params.url["thumbnail"]? - pp "#{CONFIG.thumbnails}/#{thumbnail}" begin send_file env, "#{CONFIG.thumbnails}/#{thumbnail}" diff --git a/src/routes/upload.cr b/src/routes/upload.cr index c5de027..30ce185 100644 --- a/src/routes/upload.cr +++ b/src/routes/upload.cr @@ -10,7 +10,7 @@ module Routes::Upload property id : String property ext : String property name : String - property checksum : String + property checksum : String? @[JSON::Field(key: "deleteKey")] property delete_key : String @[JSON::Field(key: "deleteLink")] @@ -31,7 +31,7 @@ module Routes::Upload def upload(env) host = env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]? scheme = env.request.headers["X-Forwarded-Proto"]? || "http" - ip_addr = env.request.headers["X-Real-IP"]? || env.request.remote_address + ip_addr = env.request.headers["X-Real-IP"]? || env.request.remote_address.as?(Socket::IPAddress).try &.address env.response.content_type = "application/json" # You can modify this if you want to allow files smaller than 1MiB. @@ -46,6 +46,7 @@ module Routes::Upload end file = UFile.new + ip = UIP.new HTTP::FormData.parse(env.request) do |upload| upload_filename = upload.filename @@ -70,16 +71,21 @@ module Routes::Upload IO.copy(upload.body, output) end - file.uploaded_at = Time.utc.to_unix.to_s - file.checksum = Utils::Hashing.hash_file(file_path) + file.uploaded_at = Time.utc.to_unix + + if CONFIG.enable_checksums + file.checksum = Utils::Hashing.hash_file(file_path) + end end + file.ip = ip_addr.to_s + ip.ip = file.ip + ip.date = file.uploaded_at + if CONFIG.deleteKeyLength > 0 file.delete_key = Random.base58(CONFIG.deleteKeyLength) end - # X-Real-IP if behind a reverse proxy and the header is set in the reverse - # proxy configuration. begin spawn { Utils.generate_thumbnail(file.filename, file.extension) } rescue ex @@ -88,10 +94,8 @@ module Routes::Upload begin Database::Files.insert(file) - # Database::IP.insert(ip_addr) - # SQL.exec "INSERT OR IGNORE INTO ips (ip, date) VALUES (?, ?)", ip_address, Time.utc.to_unix - # # SQL.exec "INSERT OR IGNORE INTO ips (ip) VALUES ('#{ip_address}')" - # SQL.exec "UPDATE ips SET count = count + 1 WHERE ip = ('#{ip_address}')" + exists = Database::IP.insert(ip).rows_affected == 0 + Database::IP.increase_count(ip) if exists rescue ex LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" ee 500, "An error ocurred when trying to insert the data into the DB" @@ -100,180 +104,4 @@ module Routes::Upload res = Response.new(file, scheme, host) res.to_json end - - # The most unoptimized and unstable feature lol - # def upload_url_bulk(env) - # env.response.content_type = "application/json" - # ip_address = Utils.ip_address(env) - # protocol = Utils.protocol(env) - # host = Utils.host(env) - # begin - # files = env.params.json["files"].as((Array(JSON::Any))) - # rescue ex : JSON::ParseException - # LOGGER.error "Body malformed: #{ex.message}" - # ee 400, "Body malformed: #{ex.message}" - # rescue ex - # LOGGER.error "Unknown error: #{ex.message}" - # ee 500, "Unknown error" - # end - # successfull_files = [] of NamedTuple(filename: String, extension: String, original_filename: String, checksum: String, delete_key: String | Nil) - # failed_files = [] of String - # # X-Real-IP if behind a reverse proxy and the header is set in the reverse - # # proxy configuration. - # files.each do |url| - # url = url.to_s - # filename = Utils.generate_filename - # original_filename = "" - # extension = "" - # checksum = "" - # uploaded_at = Time.utc - # extension = File.extname(URI.parse(url).path) - # if CONFIG.deleteKeyLength > 0 - # delete_key = Random.base58(CONFIG.deleteKeyLength) - # end - # file_path = "#{CONFIG.files}/#{filename}#{extension}" - # File.open(file_path, "w") do |output| - # begin - # HTTP::Client.get(url) do |res| - # IO.copy(res.body_io, output) - # end - # rescue ex - # LOGGER.debug "Failed to download file '#{url}': #{ex.message}" - # ee 403, "Failed to download file '#{url}'" - # failed_files << url - # end - # end - # # successfull_files << url - # # end - # if extension.empty? - # extension = Utils.detect_extension(file_path) - # File.rename(file_path, file_path + extension) - # file_path = "#{CONFIG.files}/#{filename}#{extension}" - # end - # # The second one is faster and it uses less memory - # # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last - # original_filename = url.split("/").last - # checksum = Utils::Hashing.hash_file(file_path) - # begin - # spawn { Utils.generate_thumbnail(filename, extension) } - # rescue ex - # LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" - # end - # begin - # # Insert SQL data just before returning the upload information - # SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - # original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil) - # successfull_files << {filename: filename, - # original_filename: original_filename, - # extension: extension, - # delete_key: delete_key, - # checksum: checksum} - # rescue ex - # LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" - # ee 500, "An error ocurred when trying to insert the data into the DB" - # end - # end - # json = JSON.build do |j| - # j.array do - # successfull_files.each do |fileinfo| - # j.object do - # j.field "link", "#{protocol}://#{host}/#{fileinfo[:filename]}" - # j.field "linkExt", "#{protocol}://#{host}/#{fileinfo[:filename]}#{fileinfo[:extension]}" - # j.field "id", fileinfo[:filename] - # j.field "ext", fileinfo[:extension] - # j.field "name", fileinfo[:original_filename] - # j.field "checksum", fileinfo[:checksum] - # if CONFIG.deleteKeyLength > 0 - # delete_key = Random.base58(CONFIG.deleteKeyLength) - # j.field "deleteKey", fileinfo[:delete_key] - # j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{fileinfo[:delete_key]}" - # end - # end - # end - # end - # end - # json - # end - - # def upload_url(env) - # env.response.content_type = "application/json" - # ip_address = Utils.ip_address(env) - # protocol = Utils.protocol(env) - # host = Utils.host(env) - # url = env.params.query["url"] - # successfull_files = [] of NamedTuple(filename: String, extension: String, original_filename: String, checksum: String, delete_key: String | Nil) - # failed_files = [] of String - # # X-Real-IP if behind a reverse proxy and the header is set in the reverse - # # proxy configuration. - # filename = Utils.generate_filename - # original_filename = "" - # extension = "" - # checksum = "" - # uploaded_at = Time.utc - # extension = File.extname(URI.parse(url).path) - # if CONFIG.deleteKeyLength > 0 - # delete_key = Random.base58(CONFIG.deleteKeyLength) - # end - # file_path = "#{CONFIG.files}/#{filename}#{extension}" - # File.open(file_path, "w") do |output| - # begin - # # TODO: Connect timeout to prevent possible Denial of Service to the external website spamming requests - # # https://crystal-lang.org/api/1.13.2/HTTP/Client.html#connect_timeout - # HTTP::Client.get(url) do |res| - # IO.copy(res.body_io, output) - # end - # rescue ex - # LOGGER.debug "Failed to download file '#{url}': #{ex.message}" - # ee 403, "Failed to download file '#{url}': #{ex.message}" - # failed_files << url - # end - # end - # if extension.empty? - # extension = Utils.detect_extension(file_path) - # File.rename(file_path, file_path + extension) - # file_path = "#{CONFIG.files}/#{filename}#{extension}" - # end - # # The second one is faster and it uses less memory - # # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last - # original_filename = url.split("/").last - # checksum = Utils::Hashing.hash_file(file_path) - # begin - # spawn { Utils.generate_thumbnail(filename, extension) } - # rescue ex - # LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" - # end - # begin - # # Insert SQL data just before returning the upload information - # SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - # original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil) - # successfull_files << {filename: filename, - # original_filename: original_filename, - # extension: extension, - # delete_key: delete_key, - # checksum: checksum} - # rescue ex - # LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" - # ee 500, "An error ocurred when trying to insert the data into the DB" - # end - # json = JSON.build do |j| - # j.array do - # successfull_files.each do |fileinfo| - # j.object do - # j.field "link", "#{protocol}://#{host}/#{fileinfo[:filename]}" - # j.field "linkExt", "#{protocol}://#{host}/#{fileinfo[:filename]}#{fileinfo[:extension]}" - # j.field "id", fileinfo[:filename] - # j.field "ext", fileinfo[:extension] - # j.field "name", fileinfo[:original_filename] - # j.field "checksum", fileinfo[:checksum] - # if CONFIG.deleteKeyLength > 0 - # delete_key = Random.base58(CONFIG.deleteKeyLength) - # j.field "deleteKey", fileinfo[:delete_key] - # j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{fileinfo[:delete_key]}" - # end - # end - # end - # end - # end - # json - # end end diff --git a/src/routing.cr b/src/routing.cr index 27dadf6..4331b43 100644 --- a/src/routing.cr +++ b/src/routing.cr @@ -41,24 +41,26 @@ module Routing end before_post "/upload" do |env| - begin - ip_info = SQL.query_one?("SELECT ip, count, date FROM ips WHERE ip = ?", Headers.ip_addr, as: {ip: String, count: Int32, date: Int32}) - rescue ex - LOGGER.error "Error when trying to enforce rate limits for ip #{Headers.ip_addr}: #{ex.message}" - next + ip = Headers.ip_addr + if !ip + halt env, status_code: 401, response: "X-Real-IP header not present. Contact the admin to fix this!" end + ip_info = Database::IP.select(ip) + if ip_info.nil? next end - time_since_first_upload = Time.utc.to_unix - ip_info[:date] - time_until_unban = ip_info[:date] - Time.utc.to_unix + CONFIG.rateLimitPeriod - if time_since_first_upload > CONFIG.rateLimitPeriod - SQL.exec "DELETE FROM ips WHERE ip = ?", ip_info[:ip] - end if CONFIG.filesPerIP > 0 - if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod + time_since_first_upload = Time.utc.to_unix - ip_info.date + time_until_unban = ip_info.date - Time.utc.to_unix + CONFIG.rateLimitPeriod + + if time_since_first_upload > CONFIG.rateLimitPeriod + Database::IP.delete(ip_info.ip) + end + + if ip_info.count >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod halt env, status_code: 401, response: "Rate limited! Try again in #{time_until_unban} seconds" end end @@ -69,8 +71,6 @@ module Routing get "/info/chatterino", Routes::Views, :chatterino post "/upload", Routes::Upload, :upload - # get "/upload", Routes::Upload, :upload_url - # post "/api/uploadurl", Routes::Upload, :upload_url get "/:filename", Routes::Retrieve, :retrieve_file get "/thumbnail/:thumbnail", Routes::Retrieve, :retrieve_thumbnail diff --git a/src/types/ip.cr b/src/types/ip.cr index d1cd187..0b55a6e 100644 --- a/src/types/ip.cr +++ b/src/types/ip.cr @@ -1,16 +1,16 @@ -struct IP - # Without this, this class will not be able to be used as `as: UFile` on +struct UIP + # Without this, this class will not be able to be used as `as: IP` on # SQL queries include DB::Serializable property ip : String property count : Int32 - property unix_date : Int32 + property date : Int64 def initialize( - @ip, - @count, - @unix_date, + @ip = "", + @count = 1, + @date = 0, ) end diff --git a/src/types/ufile.cr b/src/types/ufile.cr index d35c705..bc100c0 100644 --- a/src/types/ufile.cr +++ b/src/types/ufile.cr @@ -3,21 +3,21 @@ struct UFile # SQL queries include DB::Serializable - property original_filename : String = "" - property filename : String = "" - property extension : String = "" - property uploaded_at : String = "" - property checksum : String = "" - property ip : String = "" - property delete_key : String = "" + property original_filename : String + property filename : String + property extension : String + property uploaded_at : Int64 + property checksum : String? + property ip : String + property delete_key : String property thumbnail : String? def initialize( @original_filename = "", @filename = "", @extension = "", - @uploaded_at = "", - @checksum = "", + @uploaded_at = 0, + @checksum = nil, @ip = "", @delete_key = "", @thumbnail = nil, diff --git a/src/utils/hashing.cr b/src/utils/hashing.cr index 8eb096a..48a4e55 100644 --- a/src/utils/hashing.cr +++ b/src/utils/hashing.cr @@ -5,7 +5,7 @@ module Utils::Hashing Digest::SHA1.hexdigest &.file(file_path) end - def hash_io(file_path : IO) : String - Digest::SHA1.hexdigest &.update(file_path) + def hash_io(file : IO) : String + Digest::SHA1.hexdigest &.update(file) end end diff --git a/src/utils/utils.cr b/src/utils/utils.cr index d76f4aa..162fcfd 100644 --- a/src/utils/utils.cr +++ b/src/utils/utils.cr @@ -2,14 +2,70 @@ module Utils extend self def create_db - if !SQL.query_one "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='files') - AND EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='ips');", as: Bool - LOGGER.info "Creating sqlite3 database at '#{CONFIG.db}'" + files_table = <<-SQL + CREATE TABLE + IF NOT EXISTS files + ( + original_filename text not null, + filename text not null, + extension text not null, + uploaded_at integer not null, + checksum text, + ip text not null, + delete_key text not null, + thumbnail text, + PRIMARY KEY(filename) + ) + SQL + + ip_table = <<-SQL + CREATE TABLE + IF NOT EXISTS ips + ( + ip text, + count integer DEFAULT 0, + date integer, + PRIMARY KEY(ip) + ) + SQL + + files_table_check = <<-SQL + SELECT EXISTS + ( + SELECT 1 FROM + sqlite_schema + WHERE type='table' + AND name='files' + ) + SQL + + ip_table_check = <<-SQL + SELECT EXISTS + ( + SELECT 1 FROM + sqlite_schema + WHERE type='table' + AND name='ips' + ) + SQL + + files_table_exists = SQL.query_one(files_table_check, as: Bool) + ip_table_exists = SQL.query_one(ip_table_check, as: Bool) + + if (!files_table_exists) + LOGGER.info "create_db: Creating table 'files'" begin - SQL.exec "CREATE TABLE IF NOT EXISTS files - (original_filename text, filename text, extension text, uploaded_at text, checksum text, ip text, delete_key text, thumbnail text)" - SQL.exec "CREATE TABLE IF NOT EXISTS ips - (ip text UNIQUE, count integer DEFAULT 0, date integer)" + SQL.exec(files_table) + rescue ex + LOGGER.fatal "#{ex.message}" + exit(1) + end + end + + if (!ip_table_exists) + LOGGER.info "create_db: Creating table 'ips'" + begin + SQL.exec(ip_table) rescue ex LOGGER.fatal "#{ex.message}" exit(1)