Fijxu/file-uploader-crystal
1
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages 1 Wiki Activity Actions

0.8.2: Admin API, confiugurable OpenGraph user-agents.

Browse source
This commit is contained in:
Fijxu 2024-08-10 03:09:38 -04:00
parent 43dc289d3a
commit 3173a36a99
Signed by: Fijxu
GPG key ID: 32C1DDF333EDA6A4
9 changed files with 360 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -10,6 +10,7 @@ Already replaced lol.
- File deletion link (not available in frontend for now)
- Chatterino and ShareX support
- Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, can be disabled.)
- [Small Admin API](./src/handling/admin.cr) that allows you to delete files. (Needs to be enabled in the configuration)
- Unix socket support if you don't want to deal with all the TCP overhead
- Automatic protocol detection (HTTPS or HTTP)
- Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. It will depend of your traffic.

8
config/config.yml
View file

@ -3,6 +3,8 @@ thumbnails: "./thumbnails"
generate_thumbnails: false
db: "./db.sqlite3"
db_table_name: "files"
adminEnabled: false
adminApiKey: "asd"
filename_length: 3
# In MiB
size_limit: 512
@ -20,3 +22,9 @@ log_level: "debug"
blocked_extensions:
- "exe"
# List of useragents that use OpenGraph to gather file information
opengraph_useragents:
- "chatterino-api-cache/"
- "FFZBot/"
- "Twitterbot/"

3
src/config.cr
View file

@ -8,6 +8,8 @@ class Config
property generate_thumbnails : Bool = false
property db : String = "./db.sqlite3"
property db_table_name : String = "files"
property adminEnabled : Bool = false
property adminApiKey : String = ""
property incremental_filename_length : Bool = true
property filename_length : Int32 = 3
# In MiB
@ -20,6 +22,7 @@ class Config
property delete_key_length : Int32 = 4
# Blocked extensions that are not allowed to be uploaded to the server
property blocked_extensions : Array(String) = [] of String
property opengraph_useragents : Array(String) = [] of String
property siteInfo : String = "xd"
property siteWarning : String? = ""
property log_level : LogLevel = LogLevel::Info

3
src/file-uploader.cr
View file

@ -8,7 +8,7 @@ require "digest"
require "./logger"
require "./routing"
require "./utils"
require "./handling"
require "./handling/**"
require "./config"
require "./jobs"
require "./lib/**"
@ -27,6 +27,7 @@ SQL = DB.open("sqlite3://#{CONFIG.db}")
CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}
CURRENT_TAG = {{ "#{`git describe --long --abbrev=7 --tags | sed 's/([^-]*)-g.*/r\1/;s/-/./g'`.strip}" }}
Utils.check_dependencies
Utils.create_db

49
src/handling/admin.cr Normal file
View file

@ -0,0 +1,49 @@
require "../http-errors"
module Handling::Admin
extend self
def delete_file(env)
if env.request.headers.try &.["X-Api-Key"]? != CONFIG.adminApiKey || nil
error401 "Wrong API Key"
end
files = env.params.json["files"].as((Array(JSON::Any)))
successfull_files = [] of String
failed_files = [] of String
files.each do |file|
file = file.to_s
begin
fileinfo = SQL.query_one("SELECT filename, extension, thumbnail
FROM #{CONFIG.db_table_name}
WHERE filename = ?",
file,
as: {filename: String, extension: String, thumbnail: String | Nil})
# Delete file
File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}")
if fileinfo[:thumbnail]
# Delete thumbnail
File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}")
end
# Delete entry from db
SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE filename = ?", file
LOGGER.debug "File '#{fileinfo[:filename]}' was deleted"
successfull_files << file
rescue ex : DB::NoResultsError
LOGGER.error("File '#{file}' doesn't exist: #{ex.message}")
failed_files << file
rescue ex
LOGGER.error "Unknown error: #{ex.message}"
error500 "Unknown error: #{ex.message}"
end
end
json = JSON.build do |j|
j.object do
j.field "successfull", successfull_files.size
j.field "failed", failed_files.size
j.field "successfullFiles", successfull_files
j.field "failedFiles", failed_files
end
end
end
end

270
src/handling/handling.cr Normal file
View file

@ -0,0 +1,270 @@
require "../http-errors"
require "http/client"
module Handling
extend self
def upload(env)
env.response.content_type = "application/json"
# You can modify this if you want to allow files smaller than 1MiB.
# This is generally a good way to check the filesize but there is a better way to do it
# which is inspecting the file directly (If I'm not wrong).
if CONFIG.size_limit > 0
if env.request.headers["Content-Length"].to_i > 1048576*CONFIG.size_limit
error413("File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB")
end
end
filename = ""
extension = ""
original_filename = ""
uploaded_at = ""
checksum = ""
ip_address = ""
delete_key = nil
# TODO: Return the file that matches a checksum inside the database
HTTP::FormData.parse(env.request) do |upload|
if upload.filename.nil? || upload.filename.to_s.empty?
LOGGER.debug "No file provided by the user"
error403("No file provided")
end
# TODO: upload.body is emptied when is copied or read
# Utils.check_duplicate(upload.dup)
extension = File.extname("#{upload.filename}")
if CONFIG.blocked_extensions.includes?(extension.split(".")[1])
error401("Extension '#{extension}' is not allowed")
end
filename = Utils.generate_filename
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
File.open(file_path, "w") do |output|
IO.copy(upload.body, output)
end
original_filename = upload.filename
uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
checksum = Utils.hash_file(file_path)
end
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
# X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
# proxy configuration.
ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first
json = JSON.build do |j|
j.object do
j.field "link", "#{protocol}://#{host}/#{filename}"
j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
j.field "id", filename
j.field "ext", extension
j.field "name", original_filename
j.field "checksum", checksum
if CONFIG.delete_key_length > 0
delete_key = Random.base58(CONFIG.delete_key_length)
j.field "deleteKey", delete_key
j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
end
end
end
begin
spawn { Utils.generate_thumbnail(filename, extension) }
rescue ex
LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
end
begin
# Insert SQL data just before returning the upload information
SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
rescue ex
LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
error500("An error ocurred when trying to insert the data into the DB")
end
return json
end
# The most unoptimized and unstable feature lol
def upload_url(env)
env.response.content_type = "application/json"
extension = ""
filename = Utils.generate_filename
original_filename = ""
uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
checksum = ""
ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first
delete_key = nil
# X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
# proxy configuration.
if !env.params.body.nil? || env.params.body["url"].empty?
url = env.params.body["url"]
extension = File.extname(URI.parse(url).path)
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
File.open(file_path, "w") do |output|
begin
HTTP::Client.get(url) do |res|
IO.copy(res.body_io, output)
end
rescue ex
LOGGER.debug "Failed to download file '#{url}': #{ex.message}"
error403("Failed to download file '#{url}'")
end
end
if extension.empty?
extension = Utils.detect_extension(file_path)
File.rename(file_path, file_path + extension)
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
end
# TODO: Benchmark this:
# original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last
original_filename = url.split("/").last
checksum = Utils.hash_file(file_path)
if !filename.empty?
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
json = JSON.build do |j|
j.object do
j.field "link", "#{protocol}://#{host}/#{filename}"
j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
j.field "id", filename
j.field "ext", extension
j.field "name", original_filename
j.field "checksum", checksum
if CONFIG.delete_key_length > 0
delete_key = Random.base58(CONFIG.delete_key_length)
j.field "deleteKey", delete_key
j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
end
end
end
begin
spawn { Utils.generate_thumbnail(filename, extension) }
rescue ex
LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
end
begin
# Insert SQL data just before returning the upload information
SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
rescue ex
LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
error500("An error ocurred when trying to insert the data into the DB")
end
return json
end
else
end
error403("Data malformed")
end
def retrieve_file(env)
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
begin
fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail
FROM #{CONFIG.db_table_name}
WHERE filename = ?",
env.params.url["filename"].split(".").first,
as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0]
headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"})
headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"})
headers(env, {"ETag" => "#{fileinfo[:checksum]}"})
CONFIG.opengraph_useragents.each do |useragent|
if env.request.headers.try &.["User-Agent"].includes?(useragent)
env.response.content_type = "text/html"
return %(
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta property="og:title" content="#{fileinfo[:ofilename]}">
<meta property="og:url" content="#{protocol}://#{host}/#{fileinfo[:filename]}">
#{if fileinfo[:thumbnail]
%(<meta property="og:image" content="#{protocol}://#{host}/thumbnail/#{fileinfo[:filename]}.jpg">)
end}
</head>
</html>
)
end
end
send_file env, "#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:ext]}"
rescue ex
LOGGER.debug "File '#{env.params.url["filename"]}' does not exist: #{ex.message}"
error403("File '#{env.params.url["filename"]}' does not exist")
end
end
def retrieve_thumbnail(env)
begin
send_file env, "#{CONFIG.thumbnails}/#{env.params.url["thumbnail"]}"
rescue ex
LOGGER.debug "Thumbnail '#{env.params.url["thumbnail"]}' does not exist: #{ex.message}"
error403("Thumbnail '#{env.params.url["thumbnail"]}' does not exist")
end
end
def stats(env)
env.response.content_type = "application/json"
begin
json_data = JSON.build do |json|
json.object do
json.field "stats" do
json.object do
json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.db_table_name}", as: Int32
json.field "maxUploadSize", CONFIG.size_limit
json.field "thumbnailGeneration", CONFIG.generate_thumbnails
json.field "filenameLength", CONFIG.filename_length
end
end
end
end
rescue ex
LOGGER.error "Unknown error: #{ex.message}"
error500("Unknown error")
end
json_data
end
def delete_file(env)
if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.db_table_name} WHERE delete_key = ?)", env.params.query["key"], as: Bool
begin
fileinfo = SQL.query_all("SELECT filename, extension, thumbnail
FROM #{CONFIG.db_table_name}
WHERE delete_key = ?",
env.params.query["key"],
as: {filename: String, extension: String, thumbnail: String | Nil})[0]
# Delete file
File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}")
if fileinfo[:thumbnail]
# Delete thumbnail
File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}")
end
# Delete entry from db
SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"]
LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}"
msg("File '#{fileinfo[:filename]}' deleted successfully")
rescue ex
LOGGER.error("Unknown error: #{ex.message}")
error500("Unknown error")
end
else
LOGGER.debug "Key '#{env.params.query["key"]}' does not exist"
error401("Delete key '#{env.params.query["key"]}' does not exist. No files were deleted")
end
end
def sharex_config(env)
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
env.response.content_type = "application/json"
env.response.headers["Content-Disposition"] = "attachment; filename=\"#{host}.sxcu\""
return %({
"Version": "14.0.1",
"DestinationType": "ImageUploader, FileUploader",
"RequestMethod": "POST",
"RequestURL": "#{protocol}://#{host}/upload",
"Body": "MultipartFormData",
"FileFormName": "file",
"URL": "{json:link}",
"DeletionURL": "{json:deleteLink}",
"ErrorMessage": "{json:error}"
})
end
end

6
src/routing.cr
View file

@ -20,6 +20,12 @@ module Routing
Handling.upload_url(env)
end
if CONFIG.adminEnabled
post "/api/admin/delete" do |env|
Handling::Admin.delete_file(env)
end
end
get "/:filename" do |env|
Handling.retrieve_file(env)
end

21
src/utils.cr
View file

@ -29,7 +29,7 @@ module Utils
def create_thumbnails_dir
if !CONFIG.thumbnails
if !Dir.exists?("#{CONFIG.thumbnails}")
LOGGER.info "Creating thumbnaisl folder under '#{CONFIG.thumbnails}'"
LOGGER.info "Creating thumbnails folder under '#{CONFIG.thumbnails}'"
begin
Dir.mkdir("#{CONFIG.thumbnails}")
rescue ex
@ -145,6 +145,25 @@ module Utils
end
end
def delete_file(env)
fileinfo = SQL.query_all("SELECT filename, extension, thumbnail
FROM #{CONFIG.db_table_name}
WHERE delete_key = ?",
env.params.query["key"],
as: {filename: String, extension: String, thumbnail: String | Nil})[0]
# Delete file
File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}")
if fileinfo[:thumbnail]
# Delete thumbnail
File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}")
end
# Delete entry from db
SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"]
LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}"
msg("File '#{fileinfo[:filename]}' deleted successfully")
end
def detect_extension(file) : String
magic_bytes = {
".png" => "89504e470d0a1a0a",

2
src/views/index.ecr
View file

@ -22,7 +22,7 @@
<div>
<div style="text-align:center;">
<p>
<a href='./chatterino.png'>Chatterino Config</a> | <a href='./sharex.sxcu'>ShareX Config</a> | <a href='https://codeberg.org/Fijxu/file-uploader-crystal'>file-uploader-crystal (BETA <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %>)</a>
<a href='./chatterino.png'>Chatterino Config</a> | <a href='./sharex.sxcu'>ShareX Config</a> | <a href='https://codeberg.org/Fijxu/file-uploader-crystal'>file-uploader-crystal (BETA <%= CURRENT_TAG %> - <%= CURRENT_VERSION %> @ <%= CURRENT_BRANCH %>)</a>
</p>
<p>Archivos alojados: <%= files_hosted %></p>
</div>