0.9.6: Re-enable IP rate limits, add ips database logic and idk what more
This commit is contained in:
parent
8995f023ac
commit
b19c423648
13 changed files with 192 additions and 273 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
@ -7,4 +7,5 @@ data
|
||||||
torexitnodes.txt
|
torexitnodes.txt
|
||||||
files
|
files
|
||||||
thumbnails
|
thumbnails
|
||||||
db.sqlite3
|
db.sqlite3
|
||||||
|
config/config.yml
|
|
@ -1,39 +1,37 @@
|
||||||
colorize_logs: true
|
colorize_logs: true
|
||||||
|
log_level: "debug"
|
||||||
|
|
||||||
|
# File paths
|
||||||
files: "./files"
|
files: "./files"
|
||||||
thumbnails: "./thumbnails"
|
thumbnails: "./thumbnails"
|
||||||
generate_thumbnails: true
|
db: "./db.sqlite3"
|
||||||
db: "./db/db.sqlite3"
|
|
||||||
adminEnabled: true
|
# Tor
|
||||||
adminApiKey: "asd"
|
|
||||||
filename_length: 3
|
|
||||||
# In MiB
|
|
||||||
size_limit: 512
|
|
||||||
port: 8080
|
|
||||||
blockTorAddresses: true
|
blockTorAddresses: true
|
||||||
# Every hour
|
|
||||||
torExitNodesCheck: 1600
|
torExitNodesCheck: 1600
|
||||||
torExitNodesUrl: "https://check.torproject.org/exit-addresses"
|
torExitNodesUrl: "https://check.torproject.org/exit-addresses"
|
||||||
torExitNodesFile: "./torexitnodes.txt"
|
|
||||||
torMessage: "TOR IS BLOCKED!"
|
torMessage: "TOR IS BLOCKED!"
|
||||||
# Set this to 0 to disable rate limiting
|
|
||||||
|
generate_thumbnails: true
|
||||||
|
adminEnabled: true
|
||||||
|
adminApiKey: "asd"
|
||||||
|
size_limit: 512
|
||||||
|
enable_checksums: false
|
||||||
|
port: 8080
|
||||||
filesPerIP: 2
|
filesPerIP: 2
|
||||||
rateLimitPeriod: 20
|
rateLimitPeriod: 20
|
||||||
rateLimitMessage: ""
|
rateLimitMessage: ""
|
||||||
# If you define the unix socket, it will only listen on the socket and not the port.
|
filename_length: 3
|
||||||
#unix_socket: "/tmp/file-uploader.sock"
|
|
||||||
# In days
|
|
||||||
deleteFilesAfter: 7
|
deleteFilesAfter: 7
|
||||||
# In seconds
|
deleteFilesCheck: 1800
|
||||||
deleteFilesCheck: 1600
|
|
||||||
deleteKeyLength: 4
|
deleteKeyLength: 4
|
||||||
|
|
||||||
siteInfo: "Whatever you want to put here"
|
siteInfo: "Whatever you want to put here"
|
||||||
siteWarning: "WARNING!"
|
siteWarning: "WARNING!"
|
||||||
log_level: "debug"
|
|
||||||
|
|
||||||
blockedExtensions:
|
blockedExtensions:
|
||||||
- "exe"
|
- "exe"
|
||||||
|
|
||||||
# List of useragents that use OpenGraph to gather file information
|
|
||||||
opengraphUseragents:
|
opengraphUseragents:
|
||||||
- "chatterino-api-cache/"
|
- "chatterino-api-cache/"
|
||||||
- "FFZBot/"
|
- "FFZBot/"
|
||||||
|
@ -41,6 +39,5 @@ opengraphUseragents:
|
||||||
- "Synapse/"
|
- "Synapse/"
|
||||||
- "Mastodon/"
|
- "Mastodon/"
|
||||||
|
|
||||||
# You can leave it empty, or add your own domains.
|
# alternative_domains:
|
||||||
alternative_domains:
|
# - "example.com"
|
||||||
- "example.com"
|
|
||||||
|
|
|
@ -26,7 +26,7 @@ class Config
|
||||||
property adminEnabled : Bool = false
|
property adminEnabled : Bool = false
|
||||||
# The API key for admin routes. It's passed as a "X-Api-Key" header to the
|
# The API key for admin routes. It's passed as a "X-Api-Key" header to the
|
||||||
# request
|
# request
|
||||||
property adminApiKey : String? = ""
|
property adminApiKey : String? = nil
|
||||||
|
|
||||||
# Not implemented
|
# Not implemented
|
||||||
property incrementalfilename_length : Bool = true
|
property incrementalfilename_length : Bool = true
|
||||||
|
@ -34,6 +34,7 @@ class Config
|
||||||
property filename_length : Int32 = 3
|
property filename_length : Int32 = 3
|
||||||
# In MiB
|
# In MiB
|
||||||
property size_limit : Int16 = 512
|
property size_limit : Int16 = 512
|
||||||
|
property enable_checksums : Bool = true
|
||||||
|
|
||||||
# A file path where do you want to place a unix socket (THIS WILL DISABLE ACCESS
|
# A file path where do you want to place a unix socket (THIS WILL DISABLE ACCESS
|
||||||
# BY IP ADDRESS)
|
# BY IP ADDRESS)
|
||||||
|
@ -41,7 +42,7 @@ class Config
|
||||||
|
|
||||||
# True if you want this program to block IP addresses coming from the Tor
|
# True if you want this program to block IP addresses coming from the Tor
|
||||||
# network
|
# network
|
||||||
property blockTorAddresses : Bool? = false
|
property blockTorAddresses : Bool = false
|
||||||
# How often (in seconds) should this program download the exit nodes list
|
# How often (in seconds) should this program download the exit nodes list
|
||||||
property torExitNodesCheck : Int32 = 3600
|
property torExitNodesCheck : Int32 = 3600
|
||||||
# Only https://check.torproject.org/exit-addresses is supported
|
# Only https://check.torproject.org/exit-addresses is supported
|
||||||
|
@ -51,7 +52,8 @@ class Config
|
||||||
# tries to upload a file using curl or any other tool
|
# tries to upload a file using curl or any other tool
|
||||||
property torMessage : String? = "Tor is blocked!"
|
property torMessage : String? = "Tor is blocked!"
|
||||||
|
|
||||||
# How many files an IP address can upload to the server
|
# How many files an IP address can upload to the server. Setting this to 0
|
||||||
|
# disables rate limits
|
||||||
property filesPerIP : Int32 = 32
|
property filesPerIP : Int32 = 32
|
||||||
# How often is the file limit per IP reset? (in seconds)
|
# How often is the file limit per IP reset? (in seconds)
|
||||||
property rateLimitPeriod : Int32 = 600
|
property rateLimitPeriod : Int32 = 600
|
||||||
|
@ -83,14 +85,6 @@ class Config
|
||||||
# and in `/api/stats`
|
# and in `/api/stats`
|
||||||
property alternative_domains : Array(String) = [] of String
|
property alternative_domains : Array(String) = [] of String
|
||||||
|
|
||||||
def self.load
|
|
||||||
config_file = "config/config.yml"
|
|
||||||
config_yaml = File.read(config_file)
|
|
||||||
config = Config.from_yaml(config_yaml)
|
|
||||||
check_config(config)
|
|
||||||
config
|
|
||||||
end
|
|
||||||
|
|
||||||
def self.check_config(config : Config)
|
def self.check_config(config : Config)
|
||||||
if config.filename_length <= 0
|
if config.filename_length <= 0
|
||||||
puts "Config: filename_length cannot be less or equal to 0"
|
puts "Config: filename_length cannot be less or equal to 0"
|
||||||
|
@ -104,4 +98,11 @@ class Config
|
||||||
config.thumbnails = config.thumbnails.chomp('/')
|
config.thumbnails = config.thumbnails.chomp('/')
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def self.load(config_file : String = "config/config.yml")
|
||||||
|
config_yaml = File.read(config_file)
|
||||||
|
config = Config.from_yaml(config_yaml)
|
||||||
|
check_config(config)
|
||||||
|
config
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -1,16 +1,55 @@
|
||||||
module Database::IP
|
module Database::IP
|
||||||
|
extend self
|
||||||
|
|
||||||
# -------------------
|
# -------------------
|
||||||
# Insert / Delete
|
# Insert / Delete
|
||||||
# -------------------
|
# -------------------
|
||||||
|
|
||||||
def insert(ip : IP) : Nil
|
def insert(ip : UIP) : DB::ExecResult
|
||||||
request = <<-SQL
|
request = <<-SQL
|
||||||
INSERT OR IGNORE
|
INSERT OR IGNORE
|
||||||
INTO ips (ip, date)
|
INTO ips
|
||||||
VALUES ($1, $2)
|
VALUES ($1, $2, $3)
|
||||||
ON CONFLICT DO NOTHING
|
|
||||||
SQL
|
SQL
|
||||||
|
|
||||||
SQL.exec(request, *ip.to_tuple)
|
SQL.exec(request, *ip.to_tuple)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def delete(ip : String) : Nil
|
||||||
|
request = <<-SQL
|
||||||
|
DELETE
|
||||||
|
FROM ips
|
||||||
|
WHERE ip = ?
|
||||||
|
SQL
|
||||||
|
|
||||||
|
SQL.exec(request, ip)
|
||||||
|
end
|
||||||
|
|
||||||
|
# -------------------
|
||||||
|
# Select
|
||||||
|
# -------------------
|
||||||
|
|
||||||
|
def select(ip : String) : UIP?
|
||||||
|
request = <<-SQL
|
||||||
|
SELECT *
|
||||||
|
FROM ips
|
||||||
|
WHERE ip = ?
|
||||||
|
SQL
|
||||||
|
|
||||||
|
SQL.query_one?(request, ip, as: UIP)
|
||||||
|
end
|
||||||
|
|
||||||
|
# -------------------
|
||||||
|
# Update
|
||||||
|
# -------------------
|
||||||
|
|
||||||
|
def increase_count(ip : UIP) : Nil
|
||||||
|
request = <<-SQL
|
||||||
|
UPDATE ips
|
||||||
|
SET count = count + 1
|
||||||
|
WHERE ip = $1
|
||||||
|
SQL
|
||||||
|
|
||||||
|
SQL.exec(request, ip.ip)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -13,14 +13,14 @@ end
|
||||||
|
|
||||||
module Headers
|
module Headers
|
||||||
macro host
|
macro host
|
||||||
env.request.headers["X-Forwarded-Host"]?
|
env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]?
|
||||||
end
|
end
|
||||||
|
|
||||||
macro scheme
|
macro scheme
|
||||||
env.request.headers["X-Forwarded-Proto"]?
|
env.request.headers["X-Forwarded-Proto"]? || "http"
|
||||||
end
|
end
|
||||||
|
|
||||||
macro ip_addr
|
macro ip_addr
|
||||||
env.request.headers["X-Real-IP"]?
|
env.request.headers["X-Real-IP"]? || env.request.remote_address.as?(Socket::IPAddress).try &.address
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -5,7 +5,7 @@ module Routes::Deletion
|
||||||
key = env.params.query["key"]?
|
key = env.params.query["key"]?
|
||||||
|
|
||||||
if !key || key.empty?
|
if !key || key.empty?
|
||||||
ee 400, "No delete key suplied"
|
ee 400, "No delete key supplied"
|
||||||
end
|
end
|
||||||
|
|
||||||
file = Database::Files.select_with_key(key)
|
file = Database::Files.select_with_key(key)
|
||||||
|
|
|
@ -4,7 +4,7 @@ module Routes::Retrieve
|
||||||
def retrieve_file(env)
|
def retrieve_file(env)
|
||||||
host = env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]?
|
host = env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]?
|
||||||
scheme = env.request.headers["X-Forwarded-Proto"]? || "http"
|
scheme = env.request.headers["X-Forwarded-Proto"]? || "http"
|
||||||
ip_addr = env.request.headers["X-Real-IP"]? || env.request.remote_address
|
ip_addr = env.request.headers["X-Real-IP"]? || env.request.remote_address.as?(Socket::IPAddress).try &.address
|
||||||
filename = env.params.url["filename"].split(".").first
|
filename = env.params.url["filename"].split(".").first
|
||||||
|
|
||||||
begin
|
begin
|
||||||
|
@ -23,18 +23,16 @@ module Routes::Retrieve
|
||||||
CONFIG.opengraphUseragents.each do |useragent|
|
CONFIG.opengraphUseragents.each do |useragent|
|
||||||
env.response.content_type = "text/html"
|
env.response.content_type = "text/html"
|
||||||
|
|
||||||
if env.request.headers.["User-Agent"]?.try &.includes?(useragent)
|
if env.request.headers["User-Agent"]?.try &.includes?(useragent)
|
||||||
return %(
|
return %(
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta property="og:title" content="#{file.filename}">
|
<meta property="og:title" content="#{file.filename}">
|
||||||
<meta property="og:url" content="#{scheme}://#{host}/#{file.filename}">
|
<meta property="og:url" content="#{scheme}://#{host}/#{file.filename}">
|
||||||
#{if file.thumbnail
|
#{%(<meta property="og:image" content="#{scheme}://#{host}/thumbnail/#{file.filename}.jpg">) if file.thumbnail}
|
||||||
%(<meta property="og:image" content="#{scheme}://#{host}/thumbnail/#{file.filename}.jpg">)
|
</head>
|
||||||
end}
|
|
||||||
</head>
|
|
||||||
</html>
|
</html>
|
||||||
)
|
)
|
||||||
end
|
end
|
||||||
|
@ -44,7 +42,6 @@ module Routes::Retrieve
|
||||||
|
|
||||||
def retrieve_thumbnail(env)
|
def retrieve_thumbnail(env)
|
||||||
thumbnail = env.params.url["thumbnail"]?
|
thumbnail = env.params.url["thumbnail"]?
|
||||||
pp "#{CONFIG.thumbnails}/#{thumbnail}"
|
|
||||||
|
|
||||||
begin
|
begin
|
||||||
send_file env, "#{CONFIG.thumbnails}/#{thumbnail}"
|
send_file env, "#{CONFIG.thumbnails}/#{thumbnail}"
|
||||||
|
|
|
@ -10,7 +10,7 @@ module Routes::Upload
|
||||||
property id : String
|
property id : String
|
||||||
property ext : String
|
property ext : String
|
||||||
property name : String
|
property name : String
|
||||||
property checksum : String
|
property checksum : String?
|
||||||
@[JSON::Field(key: "deleteKey")]
|
@[JSON::Field(key: "deleteKey")]
|
||||||
property delete_key : String
|
property delete_key : String
|
||||||
@[JSON::Field(key: "deleteLink")]
|
@[JSON::Field(key: "deleteLink")]
|
||||||
|
@ -31,7 +31,7 @@ module Routes::Upload
|
||||||
def upload(env)
|
def upload(env)
|
||||||
host = env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]?
|
host = env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]?
|
||||||
scheme = env.request.headers["X-Forwarded-Proto"]? || "http"
|
scheme = env.request.headers["X-Forwarded-Proto"]? || "http"
|
||||||
ip_addr = env.request.headers["X-Real-IP"]? || env.request.remote_address
|
ip_addr = env.request.headers["X-Real-IP"]? || env.request.remote_address.as?(Socket::IPAddress).try &.address
|
||||||
env.response.content_type = "application/json"
|
env.response.content_type = "application/json"
|
||||||
|
|
||||||
# You can modify this if you want to allow files smaller than 1MiB.
|
# You can modify this if you want to allow files smaller than 1MiB.
|
||||||
|
@ -46,6 +46,7 @@ module Routes::Upload
|
||||||
end
|
end
|
||||||
|
|
||||||
file = UFile.new
|
file = UFile.new
|
||||||
|
ip = UIP.new
|
||||||
|
|
||||||
HTTP::FormData.parse(env.request) do |upload|
|
HTTP::FormData.parse(env.request) do |upload|
|
||||||
upload_filename = upload.filename
|
upload_filename = upload.filename
|
||||||
|
@ -70,16 +71,21 @@ module Routes::Upload
|
||||||
IO.copy(upload.body, output)
|
IO.copy(upload.body, output)
|
||||||
end
|
end
|
||||||
|
|
||||||
file.uploaded_at = Time.utc.to_unix.to_s
|
file.uploaded_at = Time.utc.to_unix
|
||||||
file.checksum = Utils::Hashing.hash_file(file_path)
|
|
||||||
|
if CONFIG.enable_checksums
|
||||||
|
file.checksum = Utils::Hashing.hash_file(file_path)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
file.ip = ip_addr.to_s
|
||||||
|
ip.ip = file.ip
|
||||||
|
ip.date = file.uploaded_at
|
||||||
|
|
||||||
if CONFIG.deleteKeyLength > 0
|
if CONFIG.deleteKeyLength > 0
|
||||||
file.delete_key = Random.base58(CONFIG.deleteKeyLength)
|
file.delete_key = Random.base58(CONFIG.deleteKeyLength)
|
||||||
end
|
end
|
||||||
|
|
||||||
# X-Real-IP if behind a reverse proxy and the header is set in the reverse
|
|
||||||
# proxy configuration.
|
|
||||||
begin
|
begin
|
||||||
spawn { Utils.generate_thumbnail(file.filename, file.extension) }
|
spawn { Utils.generate_thumbnail(file.filename, file.extension) }
|
||||||
rescue ex
|
rescue ex
|
||||||
|
@ -88,10 +94,8 @@ module Routes::Upload
|
||||||
|
|
||||||
begin
|
begin
|
||||||
Database::Files.insert(file)
|
Database::Files.insert(file)
|
||||||
# Database::IP.insert(ip_addr)
|
exists = Database::IP.insert(ip).rows_affected == 0
|
||||||
# SQL.exec "INSERT OR IGNORE INTO ips (ip, date) VALUES (?, ?)", ip_address, Time.utc.to_unix
|
Database::IP.increase_count(ip) if exists
|
||||||
# # SQL.exec "INSERT OR IGNORE INTO ips (ip) VALUES ('#{ip_address}')"
|
|
||||||
# SQL.exec "UPDATE ips SET count = count + 1 WHERE ip = ('#{ip_address}')"
|
|
||||||
rescue ex
|
rescue ex
|
||||||
LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
|
LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
|
||||||
ee 500, "An error ocurred when trying to insert the data into the DB"
|
ee 500, "An error ocurred when trying to insert the data into the DB"
|
||||||
|
@ -100,180 +104,4 @@ module Routes::Upload
|
||||||
res = Response.new(file, scheme, host)
|
res = Response.new(file, scheme, host)
|
||||||
res.to_json
|
res.to_json
|
||||||
end
|
end
|
||||||
|
|
||||||
# The most unoptimized and unstable feature lol
|
|
||||||
# def upload_url_bulk(env)
|
|
||||||
# env.response.content_type = "application/json"
|
|
||||||
# ip_address = Utils.ip_address(env)
|
|
||||||
# protocol = Utils.protocol(env)
|
|
||||||
# host = Utils.host(env)
|
|
||||||
# begin
|
|
||||||
# files = env.params.json["files"].as((Array(JSON::Any)))
|
|
||||||
# rescue ex : JSON::ParseException
|
|
||||||
# LOGGER.error "Body malformed: #{ex.message}"
|
|
||||||
# ee 400, "Body malformed: #{ex.message}"
|
|
||||||
# rescue ex
|
|
||||||
# LOGGER.error "Unknown error: #{ex.message}"
|
|
||||||
# ee 500, "Unknown error"
|
|
||||||
# end
|
|
||||||
# successfull_files = [] of NamedTuple(filename: String, extension: String, original_filename: String, checksum: String, delete_key: String | Nil)
|
|
||||||
# failed_files = [] of String
|
|
||||||
# # X-Real-IP if behind a reverse proxy and the header is set in the reverse
|
|
||||||
# # proxy configuration.
|
|
||||||
# files.each do |url|
|
|
||||||
# url = url.to_s
|
|
||||||
# filename = Utils.generate_filename
|
|
||||||
# original_filename = ""
|
|
||||||
# extension = ""
|
|
||||||
# checksum = ""
|
|
||||||
# uploaded_at = Time.utc
|
|
||||||
# extension = File.extname(URI.parse(url).path)
|
|
||||||
# if CONFIG.deleteKeyLength > 0
|
|
||||||
# delete_key = Random.base58(CONFIG.deleteKeyLength)
|
|
||||||
# end
|
|
||||||
# file_path = "#{CONFIG.files}/#{filename}#{extension}"
|
|
||||||
# File.open(file_path, "w") do |output|
|
|
||||||
# begin
|
|
||||||
# HTTP::Client.get(url) do |res|
|
|
||||||
# IO.copy(res.body_io, output)
|
|
||||||
# end
|
|
||||||
# rescue ex
|
|
||||||
# LOGGER.debug "Failed to download file '#{url}': #{ex.message}"
|
|
||||||
# ee 403, "Failed to download file '#{url}'"
|
|
||||||
# failed_files << url
|
|
||||||
# end
|
|
||||||
# end
|
|
||||||
# # successfull_files << url
|
|
||||||
# # end
|
|
||||||
# if extension.empty?
|
|
||||||
# extension = Utils.detect_extension(file_path)
|
|
||||||
# File.rename(file_path, file_path + extension)
|
|
||||||
# file_path = "#{CONFIG.files}/#{filename}#{extension}"
|
|
||||||
# end
|
|
||||||
# # The second one is faster and it uses less memory
|
|
||||||
# # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last
|
|
||||||
# original_filename = url.split("/").last
|
|
||||||
# checksum = Utils::Hashing.hash_file(file_path)
|
|
||||||
# begin
|
|
||||||
# spawn { Utils.generate_thumbnail(filename, extension) }
|
|
||||||
# rescue ex
|
|
||||||
# LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
|
|
||||||
# end
|
|
||||||
# begin
|
|
||||||
# # Insert SQL data just before returning the upload information
|
|
||||||
# SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
|
||||||
# original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil)
|
|
||||||
# successfull_files << {filename: filename,
|
|
||||||
# original_filename: original_filename,
|
|
||||||
# extension: extension,
|
|
||||||
# delete_key: delete_key,
|
|
||||||
# checksum: checksum}
|
|
||||||
# rescue ex
|
|
||||||
# LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
|
|
||||||
# ee 500, "An error ocurred when trying to insert the data into the DB"
|
|
||||||
# end
|
|
||||||
# end
|
|
||||||
# json = JSON.build do |j|
|
|
||||||
# j.array do
|
|
||||||
# successfull_files.each do |fileinfo|
|
|
||||||
# j.object do
|
|
||||||
# j.field "link", "#{protocol}://#{host}/#{fileinfo[:filename]}"
|
|
||||||
# j.field "linkExt", "#{protocol}://#{host}/#{fileinfo[:filename]}#{fileinfo[:extension]}"
|
|
||||||
# j.field "id", fileinfo[:filename]
|
|
||||||
# j.field "ext", fileinfo[:extension]
|
|
||||||
# j.field "name", fileinfo[:original_filename]
|
|
||||||
# j.field "checksum", fileinfo[:checksum]
|
|
||||||
# if CONFIG.deleteKeyLength > 0
|
|
||||||
# delete_key = Random.base58(CONFIG.deleteKeyLength)
|
|
||||||
# j.field "deleteKey", fileinfo[:delete_key]
|
|
||||||
# j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{fileinfo[:delete_key]}"
|
|
||||||
# end
|
|
||||||
# end
|
|
||||||
# end
|
|
||||||
# end
|
|
||||||
# end
|
|
||||||
# json
|
|
||||||
# end
|
|
||||||
|
|
||||||
# def upload_url(env)
|
|
||||||
# env.response.content_type = "application/json"
|
|
||||||
# ip_address = Utils.ip_address(env)
|
|
||||||
# protocol = Utils.protocol(env)
|
|
||||||
# host = Utils.host(env)
|
|
||||||
# url = env.params.query["url"]
|
|
||||||
# successfull_files = [] of NamedTuple(filename: String, extension: String, original_filename: String, checksum: String, delete_key: String | Nil)
|
|
||||||
# failed_files = [] of String
|
|
||||||
# # X-Real-IP if behind a reverse proxy and the header is set in the reverse
|
|
||||||
# # proxy configuration.
|
|
||||||
# filename = Utils.generate_filename
|
|
||||||
# original_filename = ""
|
|
||||||
# extension = ""
|
|
||||||
# checksum = ""
|
|
||||||
# uploaded_at = Time.utc
|
|
||||||
# extension = File.extname(URI.parse(url).path)
|
|
||||||
# if CONFIG.deleteKeyLength > 0
|
|
||||||
# delete_key = Random.base58(CONFIG.deleteKeyLength)
|
|
||||||
# end
|
|
||||||
# file_path = "#{CONFIG.files}/#{filename}#{extension}"
|
|
||||||
# File.open(file_path, "w") do |output|
|
|
||||||
# begin
|
|
||||||
# # TODO: Connect timeout to prevent possible Denial of Service to the external website spamming requests
|
|
||||||
# # https://crystal-lang.org/api/1.13.2/HTTP/Client.html#connect_timeout
|
|
||||||
# HTTP::Client.get(url) do |res|
|
|
||||||
# IO.copy(res.body_io, output)
|
|
||||||
# end
|
|
||||||
# rescue ex
|
|
||||||
# LOGGER.debug "Failed to download file '#{url}': #{ex.message}"
|
|
||||||
# ee 403, "Failed to download file '#{url}': #{ex.message}"
|
|
||||||
# failed_files << url
|
|
||||||
# end
|
|
||||||
# end
|
|
||||||
# if extension.empty?
|
|
||||||
# extension = Utils.detect_extension(file_path)
|
|
||||||
# File.rename(file_path, file_path + extension)
|
|
||||||
# file_path = "#{CONFIG.files}/#{filename}#{extension}"
|
|
||||||
# end
|
|
||||||
# # The second one is faster and it uses less memory
|
|
||||||
# # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last
|
|
||||||
# original_filename = url.split("/").last
|
|
||||||
# checksum = Utils::Hashing.hash_file(file_path)
|
|
||||||
# begin
|
|
||||||
# spawn { Utils.generate_thumbnail(filename, extension) }
|
|
||||||
# rescue ex
|
|
||||||
# LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
|
|
||||||
# end
|
|
||||||
# begin
|
|
||||||
# # Insert SQL data just before returning the upload information
|
|
||||||
# SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
|
||||||
# original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil)
|
|
||||||
# successfull_files << {filename: filename,
|
|
||||||
# original_filename: original_filename,
|
|
||||||
# extension: extension,
|
|
||||||
# delete_key: delete_key,
|
|
||||||
# checksum: checksum}
|
|
||||||
# rescue ex
|
|
||||||
# LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
|
|
||||||
# ee 500, "An error ocurred when trying to insert the data into the DB"
|
|
||||||
# end
|
|
||||||
# json = JSON.build do |j|
|
|
||||||
# j.array do
|
|
||||||
# successfull_files.each do |fileinfo|
|
|
||||||
# j.object do
|
|
||||||
# j.field "link", "#{protocol}://#{host}/#{fileinfo[:filename]}"
|
|
||||||
# j.field "linkExt", "#{protocol}://#{host}/#{fileinfo[:filename]}#{fileinfo[:extension]}"
|
|
||||||
# j.field "id", fileinfo[:filename]
|
|
||||||
# j.field "ext", fileinfo[:extension]
|
|
||||||
# j.field "name", fileinfo[:original_filename]
|
|
||||||
# j.field "checksum", fileinfo[:checksum]
|
|
||||||
# if CONFIG.deleteKeyLength > 0
|
|
||||||
# delete_key = Random.base58(CONFIG.deleteKeyLength)
|
|
||||||
# j.field "deleteKey", fileinfo[:delete_key]
|
|
||||||
# j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{fileinfo[:delete_key]}"
|
|
||||||
# end
|
|
||||||
# end
|
|
||||||
# end
|
|
||||||
# end
|
|
||||||
# end
|
|
||||||
# json
|
|
||||||
# end
|
|
||||||
end
|
end
|
||||||
|
|
|
@ -41,24 +41,26 @@ module Routing
|
||||||
end
|
end
|
||||||
|
|
||||||
before_post "/upload" do |env|
|
before_post "/upload" do |env|
|
||||||
begin
|
ip = Headers.ip_addr
|
||||||
ip_info = SQL.query_one?("SELECT ip, count, date FROM ips WHERE ip = ?", Headers.ip_addr, as: {ip: String, count: Int32, date: Int32})
|
if !ip
|
||||||
rescue ex
|
halt env, status_code: 401, response: "X-Real-IP header not present. Contact the admin to fix this!"
|
||||||
LOGGER.error "Error when trying to enforce rate limits for ip #{Headers.ip_addr}: #{ex.message}"
|
|
||||||
next
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
ip_info = Database::IP.select(ip)
|
||||||
|
|
||||||
if ip_info.nil?
|
if ip_info.nil?
|
||||||
next
|
next
|
||||||
end
|
end
|
||||||
|
|
||||||
time_since_first_upload = Time.utc.to_unix - ip_info[:date]
|
|
||||||
time_until_unban = ip_info[:date] - Time.utc.to_unix + CONFIG.rateLimitPeriod
|
|
||||||
if time_since_first_upload > CONFIG.rateLimitPeriod
|
|
||||||
SQL.exec "DELETE FROM ips WHERE ip = ?", ip_info[:ip]
|
|
||||||
end
|
|
||||||
if CONFIG.filesPerIP > 0
|
if CONFIG.filesPerIP > 0
|
||||||
if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod
|
time_since_first_upload = Time.utc.to_unix - ip_info.date
|
||||||
|
time_until_unban = ip_info.date - Time.utc.to_unix + CONFIG.rateLimitPeriod
|
||||||
|
|
||||||
|
if time_since_first_upload > CONFIG.rateLimitPeriod
|
||||||
|
Database::IP.delete(ip_info.ip)
|
||||||
|
end
|
||||||
|
|
||||||
|
if ip_info.count >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod
|
||||||
halt env, status_code: 401, response: "Rate limited! Try again in #{time_until_unban} seconds"
|
halt env, status_code: 401, response: "Rate limited! Try again in #{time_until_unban} seconds"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -69,8 +71,6 @@ module Routing
|
||||||
get "/info/chatterino", Routes::Views, :chatterino
|
get "/info/chatterino", Routes::Views, :chatterino
|
||||||
|
|
||||||
post "/upload", Routes::Upload, :upload
|
post "/upload", Routes::Upload, :upload
|
||||||
# get "/upload", Routes::Upload, :upload_url
|
|
||||||
# post "/api/uploadurl", Routes::Upload, :upload_url
|
|
||||||
|
|
||||||
get "/:filename", Routes::Retrieve, :retrieve_file
|
get "/:filename", Routes::Retrieve, :retrieve_file
|
||||||
get "/thumbnail/:thumbnail", Routes::Retrieve, :retrieve_thumbnail
|
get "/thumbnail/:thumbnail", Routes::Retrieve, :retrieve_thumbnail
|
||||||
|
|
|
@ -1,16 +1,16 @@
|
||||||
struct IP
|
struct UIP
|
||||||
# Without this, this class will not be able to be used as `as: UFile` on
|
# Without this, this class will not be able to be used as `as: IP` on
|
||||||
# SQL queries
|
# SQL queries
|
||||||
include DB::Serializable
|
include DB::Serializable
|
||||||
|
|
||||||
property ip : String
|
property ip : String
|
||||||
property count : Int32
|
property count : Int32
|
||||||
property unix_date : Int32
|
property date : Int64
|
||||||
|
|
||||||
def initialize(
|
def initialize(
|
||||||
@ip,
|
@ip = "",
|
||||||
@count,
|
@count = 1,
|
||||||
@unix_date,
|
@date = 0,
|
||||||
)
|
)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -3,21 +3,21 @@ struct UFile
|
||||||
# SQL queries
|
# SQL queries
|
||||||
include DB::Serializable
|
include DB::Serializable
|
||||||
|
|
||||||
property original_filename : String = ""
|
property original_filename : String
|
||||||
property filename : String = ""
|
property filename : String
|
||||||
property extension : String = ""
|
property extension : String
|
||||||
property uploaded_at : String = ""
|
property uploaded_at : Int64
|
||||||
property checksum : String = ""
|
property checksum : String?
|
||||||
property ip : String = ""
|
property ip : String
|
||||||
property delete_key : String = ""
|
property delete_key : String
|
||||||
property thumbnail : String?
|
property thumbnail : String?
|
||||||
|
|
||||||
def initialize(
|
def initialize(
|
||||||
@original_filename = "",
|
@original_filename = "",
|
||||||
@filename = "",
|
@filename = "",
|
||||||
@extension = "",
|
@extension = "",
|
||||||
@uploaded_at = "",
|
@uploaded_at = 0,
|
||||||
@checksum = "",
|
@checksum = nil,
|
||||||
@ip = "",
|
@ip = "",
|
||||||
@delete_key = "",
|
@delete_key = "",
|
||||||
@thumbnail = nil,
|
@thumbnail = nil,
|
||||||
|
|
|
@ -5,7 +5,7 @@ module Utils::Hashing
|
||||||
Digest::SHA1.hexdigest &.file(file_path)
|
Digest::SHA1.hexdigest &.file(file_path)
|
||||||
end
|
end
|
||||||
|
|
||||||
def hash_io(file_path : IO) : String
|
def hash_io(file : IO) : String
|
||||||
Digest::SHA1.hexdigest &.update(file_path)
|
Digest::SHA1.hexdigest &.update(file)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -2,14 +2,70 @@ module Utils
|
||||||
extend self
|
extend self
|
||||||
|
|
||||||
def create_db
|
def create_db
|
||||||
if !SQL.query_one "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='files')
|
files_table = <<-SQL
|
||||||
AND EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='ips');", as: Bool
|
CREATE TABLE
|
||||||
LOGGER.info "Creating sqlite3 database at '#{CONFIG.db}'"
|
IF NOT EXISTS files
|
||||||
|
(
|
||||||
|
original_filename text not null,
|
||||||
|
filename text not null,
|
||||||
|
extension text not null,
|
||||||
|
uploaded_at integer not null,
|
||||||
|
checksum text,
|
||||||
|
ip text not null,
|
||||||
|
delete_key text not null,
|
||||||
|
thumbnail text,
|
||||||
|
PRIMARY KEY(filename)
|
||||||
|
)
|
||||||
|
SQL
|
||||||
|
|
||||||
|
ip_table = <<-SQL
|
||||||
|
CREATE TABLE
|
||||||
|
IF NOT EXISTS ips
|
||||||
|
(
|
||||||
|
ip text,
|
||||||
|
count integer DEFAULT 0,
|
||||||
|
date integer,
|
||||||
|
PRIMARY KEY(ip)
|
||||||
|
)
|
||||||
|
SQL
|
||||||
|
|
||||||
|
files_table_check = <<-SQL
|
||||||
|
SELECT EXISTS
|
||||||
|
(
|
||||||
|
SELECT 1 FROM
|
||||||
|
sqlite_schema
|
||||||
|
WHERE type='table'
|
||||||
|
AND name='files'
|
||||||
|
)
|
||||||
|
SQL
|
||||||
|
|
||||||
|
ip_table_check = <<-SQL
|
||||||
|
SELECT EXISTS
|
||||||
|
(
|
||||||
|
SELECT 1 FROM
|
||||||
|
sqlite_schema
|
||||||
|
WHERE type='table'
|
||||||
|
AND name='ips'
|
||||||
|
)
|
||||||
|
SQL
|
||||||
|
|
||||||
|
files_table_exists = SQL.query_one(files_table_check, as: Bool)
|
||||||
|
ip_table_exists = SQL.query_one(ip_table_check, as: Bool)
|
||||||
|
|
||||||
|
if (!files_table_exists)
|
||||||
|
LOGGER.info "create_db: Creating table 'files'"
|
||||||
begin
|
begin
|
||||||
SQL.exec "CREATE TABLE IF NOT EXISTS files
|
SQL.exec(files_table)
|
||||||
(original_filename text, filename text, extension text, uploaded_at text, checksum text, ip text, delete_key text, thumbnail text)"
|
rescue ex
|
||||||
SQL.exec "CREATE TABLE IF NOT EXISTS ips
|
LOGGER.fatal "#{ex.message}"
|
||||||
(ip text UNIQUE, count integer DEFAULT 0, date integer)"
|
exit(1)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if (!ip_table_exists)
|
||||||
|
LOGGER.info "create_db: Creating table 'ips'"
|
||||||
|
begin
|
||||||
|
SQL.exec(ip_table)
|
||||||
rescue ex
|
rescue ex
|
||||||
LOGGER.fatal "#{ex.message}"
|
LOGGER.fatal "#{ex.message}"
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
Loading…
Add table
Reference in a new issue