Fijxu/file-uploader-crystal
1
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages 1 Wiki Activity Actions

0.9.0: Mode admin endpoints, use UTC date again since sqlite doesn't support HTTP dates, use official tor exit nodes list

Browse source
This commit is contained in:
Fijxu 2024-08-25 18:08:56 -04:00
parent 7ee5956970
commit cbeba2b0a2
Signed by: Fijxu
GPG key ID: 32C1DDF333EDA6A4
7 changed files with 164 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
config/config.yml
View file

@ -1,6 +1,6 @@
files: "./files"
thumbnails: "./thumbnails"
generateThumbnails: false
generateThumbnails: true
db: "./db.sqlite3"
dbTableName: "files"
adminEnabled: true
@ -11,8 +11,8 @@ size_limit: 512
port: 8080
blockTorAddresses: true
# Every hour
torExitNodesCheck: 3600
torExitNodesUrl: "https://www.dan.me.uk/torlist/?exit"
torExitNodesCheck: 1600
torExitNodesUrl: "https://check.torproject.org/exit-addresses"
torExitNodesFile: "./torexitnodes.txt"
torMessage: "TOR IS BLOCKED!"
filesPerIP: 2
@ -22,9 +22,9 @@ rateLimitMessage: ""
# If you define the unix socket, it will only listen on the socket and not the port.
#unix_socket: "/tmp/file-uploader.sock"
# In days
deleteFilesAfter: 7
deleteFilesAfter: 1
# In seconds
deleteFilesCheck: 1800
deleteFilesCheck: 1600
deleteKeyLength: 4
siteInfo: "Whatever you want to put here"
siteWarning: "WARNING!"

5
src/config.cr
View file

@ -35,9 +35,8 @@ class Config
property blockTorAddresses : Bool? = false
# How often (in seconds) should this program download the exit nodes list
property torExitNodesCheck : Int32 = 3600
# A URL with a list of exit nodes addresses
# The list needs to contain a IP address per line
property torExitNodesUrl : String = "https://www.dan.me.uk/torlist/?exit"
# Only https://check.torproject.org/exit-addresses is supported
property torExitNodesUrl : String = "https://check.torproject.org/exit-addresses"
# Where the file of the exit nodes will be located, can be placed anywhere
property torExitNodesFile : String = "./torexitnodes.txt"
# Message that will be displayed to the Tor user.

125
src/handling/admin.cr
View file

@ -3,7 +3,12 @@ require "../http-errors"
module Handling::Admin
extend self
# private macro json_fill(named_tuple, field_name)
# j.field {{field_name}}, {{named_tuple}}[:{{field_name}}]
# end
# /api/admin/delete
# curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: asd" http://localhost:8080/api/admin/delete -d '{"files": ["j63"]}' | jq
def delete_file(env)
files = env.params.json["files"].as((Array(JSON::Any)))
successfull_files = [] of String
@ -46,20 +51,22 @@ module Handling::Admin
end
# /api/admin/deleteiplimit
# curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: asd" http://localhost:8080/api/admin/deleteiplimit -d '{"ips": ["127.0.0.1"]}' | jq
def delete_ip_limit(env)
ips = env.params.json["ips"].as((Array(JSON::Any)))
successfull_ips = [] of String
failed_ips = [] of String
ips.each do |ip|
ip = ip.to_s
data = env.params.json["ips"].as((Array(JSON::Any)))
successfull = [] of String
failed = [] of String
data.each do |item|
item = item.to_s
begin
# Delete entry from db
SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", ip
LOGGER.debug "Rate limit for '#{ip}' was deleted"
successfull_ips << ip
SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", item
LOGGER.debug "Rate limit for '#{item}' was deleted"
successfull << item
rescue ex : DB::NoResultsError
LOGGER.error("Rate limit for '#{ip}' doesn't exist or is not registered in the database: #{ex.message}")
failed_ips << ip
LOGGER.error("Rate limit for '#{item}' doesn't exist or is not registered in the database: #{ex.message}")
failed << item
rescue ex
LOGGER.error "Unknown error: #{ex.message}"
error500 "Unknown error: #{ex.message}"
@ -67,11 +74,101 @@ module Handling::Admin
end
json = JSON.build do |j|
j.object do
j.field "successfull", successfull_ips.size
j.field "failed", failed_ips.size
j.field "successfullUnbans", successfull_ips
j.field "failedUnbans", failed_ips
j.field "successfull", successfull.size
j.field "failed", failed.size
j.field "successfullUnbans", successfull
j.field "failedUnbans", failed
end
end
end
# /api/admin/fileinfo
# curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: asd" http://localhost:8080/api/admin/fileinfo -d '{"files": ["j63"]}' | jq
def retrieve_file_info(env)
data = env.params.json["files"].as((Array(JSON::Any)))
successfull = [] of NamedTuple(original_filename: String, filename: String, extension: String,
uploaded_at: String, checksum: String, ip: String, delete_key: String,
thumbnail: String | Nil)
failed = [] of String
data.each do |item|
item = item.to_s
begin
fileinfo = SQL.query_one("SELECT original_filename, filename, extension,
uploaded_at, checksum, ip, delete_key, thumbnail
FROM #{CONFIG.dbTableName}
WHERE filename = ?",
item,
as: {original_filename: String, filename: String, extension: String,
uploaded_at: String, checksum: String, ip: String, delete_key: String,
thumbnail: String | Nil})
successfull << fileinfo
rescue ex : DB::NoResultsError
LOGGER.error("File '#{item}' is not registered in the database: #{ex.message}")
failed << item
rescue ex
LOGGER.error "Unknown error: #{ex.message}"
error500 "Unknown error: #{ex.message}"
end
end
json = JSON.build do |j|
j.object do
j.field "files" do
j.array do
successfull.each do |fileinfo|
j.object do
j.field fileinfo[:filename] do
j.object do
j.field "original_filename", fileinfo[:original_filename]
j.field "filename", fileinfo[:filename]
j.field "extension", fileinfo[:extension]
j.field "uploaded_at", fileinfo[:uploaded_at]
j.field "checksum", fileinfo[:checksum]
j.field "ip", fileinfo[:ip]
j.field "delete_key", fileinfo[:delete_key]
j.field "thumbnail", fileinfo[:thumbnail]
end
end
end
end
end
end
j.field "successfull", successfull.size
j.field "failed", failed.size
# j.field "successfullFiles"
j.field "failedFiles", failed
end
end
end
# /api/admin/torexitnodes
# curl -X GET -H "X-Api-Key: asd" http://localhost:8080/api/admin/torexitnodes | jq
def retrieve_tor_exit_nodes(env, nodes)
json = JSON.build do |j|
j.object do
j.field "ips", nodes
end
end
end
# /api/admin/whitelist
# curl -X GET -H "X-Api-Key: asd" http://localhost:8080/api/admin/torexitnodes | jq
# def add_ip_to_whitelist(env, nodes)
# json = JSON.build do |j|
# j.object do
# j.field "ips", nodes
# end
# end
# end
# /api/admin/blacklist
# curl -X GET -H "X-Api-Key: asd" http://localhost:8080/api/admin/torexitnodes | jq
def add_ip_to_blacklist(env, nodes)
json = JSON.build do |j|
j.object do
j.field "ips", nodes
end
end
end
# MODULE END
end

26
src/handling/handling.cr
View file

@ -1,5 +1,6 @@
require "../http-errors"
require "http/client"
require "benchmark"
module Handling
extend self
@ -43,7 +44,7 @@ module Handling
IO.copy(upload.body, output)
end
original_filename = upload.filename
uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
uploaded_at = Time.utc
checksum = Utils.hash_file(file_path)
end
# X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
@ -101,7 +102,7 @@ module Handling
original_filename = ""
extension = ""
checksum = ""
uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
uploaded_at = Time.utc
extension = File.extname(URI.parse(url).path)
delete_key = nil
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
@ -123,7 +124,7 @@ module Handling
File.rename(file_path, file_path + extension)
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
end
# TODO: Benchmark this:
# The second one is faster and it uses less memory
# original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last
original_filename = url.split("/").last
checksum = Utils.hash_file(file_path)
@ -177,10 +178,21 @@ module Handling
WHERE filename = ?",
env.params.url["filename"].split(".").first,
as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0]
headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"})
headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"})
headers(env, {"ETag" => "#{fileinfo[:checksum]}"})
# Benchmark.ips do |x|
# x.report("header multiple") { headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}",
# "Last-Modified" => "#{fileinfo[:up_at]}",
# "ETag" => "#{fileinfo[:checksum]}"}) }
# x.report("shorter sleep") do
# env.response.headers["Content-Disposition"] = "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"
# env.response.headers["Last-Modified"] = "#{fileinfo[:up_at]}"
# env.response.headers["ETag"] = "#{fileinfo[:checksum]}"
# end
# end
# `env.response.headers` is faster than `headers(env, Hash(String, String))`
# https://github.com/kemalcr/kemal/blob/3243b8e0e03568ad3bd9f0ad6f445c871605b821/src/kemal/helpers/helpers.cr#L102C1-L104C4
env.response.headers["Content-Disposition"] = "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"
# env.response.headers["Last-Modified"] = "#{fileinfo[:up_at]}"
env.response.headers["ETag"] = "#{fileinfo[:checksum]}"
CONFIG.opengraphUseragents.each do |useragent|
if env.request.headers.try &.["User-Agent"].includes?(useragent)

9
src/routing.cr
View file

@ -7,6 +7,7 @@ module Routing
def reload_exit_nodes
LOGGER.debug "Updating Tor exit nodes array"
@@exit_nodes = Utils.load_tor_exit_nodes
LOGGER.debug "IPs inside the exit nodes array: #{@@exit_nodes.size}"
end
before_post "/api/admin/*" do |env|
@ -94,4 +95,12 @@ module Routing
post "/api/admin/deleteiplimit" do |env|
Handling::Admin.delete_ip_limit(env)
end
post "/api/admin/fileinfo" do |env|
Handling::Admin.retrieve_file_info(env)
end
get "/api/admin/torexitnodes" do |env|
Handling::Admin.retrieve_tor_exit_nodes(env, @@exit_nodes)
end
end

7
src/utils.cr
View file

@ -235,6 +235,13 @@ module Utils
def load_tor_exit_nodes
exit_nodes = File.read_lines(CONFIG.torExitNodesFile)
ips = [] of String
exit_nodes.each do |line|
if line.includes?("ExitAddress")
ips << line.split(" ")[1]
end
end
return ips
end
def ip_address(env) : String