Fijxu/file-uploader-crystal
1
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages 1 Wiki Activity Actions

0.8.1: Option to disable thumbnail generation, OpenGraph implementation

Browse source 
for clients that support it, dynamic ShareX configuration and more.
This commit is contained in:
Fijxu 2024-08-10 00:58:31 -04:00
parent 2058c600bc
commit 43dc289d3a
Signed by: Fijxu
GPG key ID: 32C1DDF333EDA6A4
8 changed files with 234 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -9,7 +9,7 @@ Already replaced lol.
- Temporary file uploads like Uguu
- File deletion link (not available in frontend for now)
- Chatterino and ShareX support
- Video Thumbnails for Chatterino and FrankerFaceZ
- Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, can be disabled.)
- Unix socket support if you don't want to deal with all the TCP overhead
- Automatic protocol detection (HTTPS or HTTP)
- Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. It will depend of your traffic.

2
config/config.yml
View file

@ -1,4 +1,6 @@
files: "./files"
thumbnails: "./thumbnails"
generate_thumbnails: false
db: "./db.sqlite3"
db_table_name: "files"
filename_length: 3

11
public/sharex.sxcu
View file

@ -1,11 +0,0 @@
{
"Version": "14.0.1",
"DestinationType": "ImageUploader, FileUploader",
"RequestMethod": "POST",
"RequestURL": "https://ayaya.beauty/upload",
"Body": "MultipartFormData",
"FileFormName": "file",
"URL": "{json:link}",
"DeletionURL": "{json:deleteLink}",
"ErrorMessage": "{json:error}"
}

2
src/config.cr
View file

@ -5,8 +5,10 @@ class Config
property files : String = "./files"
property thumbnails : String = "./thumbnails"
property generate_thumbnails : Bool = false
property db : String = "./db.sqlite3"
property db_table_name : String = "files"
property incremental_filename_length : Bool = true
property filename_length : Int32 = 3
# In MiB
property size_limit : Int16 = 512

3
src/file-uploader.cr
View file

@ -28,6 +28,7 @@ CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}
Utils.check_dependencies
Utils.create_db
Utils.create_files_dir
Routing.register_all
@ -39,7 +40,6 @@ Jobs.run
Kemal.config.env = "production" if !ENV.has_key?("KEMAL_ENV")
{% end %}
# Set permissions to 777 so NGINX can read and write to it (BROKEN)
if !CONFIG.unix_socket.nil?
sleep 1.second
LOGGER.info "Changing socket permissions to 777"
@ -47,6 +47,7 @@ if !CONFIG.unix_socket.nil?
File.chmod("#{CONFIG.unix_socket}", File::Permissions::All)
rescue ex
LOGGER.fatal "#{ex.message}"
exit(1)
end
end

139
src/handling.cr
View file

@ -1,4 +1,5 @@
require "./http-errors"
require "http/client"
module Handling
extend self
@ -22,7 +23,10 @@ module Handling
delete_key = nil
# TODO: Return the file that matches a checksum inside the database
HTTP::FormData.parse(env.request) do |upload|
next if upload.filename.nil? || upload.filename.to_s.empty?
if upload.filename.nil? || upload.filename.to_s.empty?
LOGGER.debug "No file provided by the user"
error403("No file provided")
end
# TODO: upload.body is emptied when is copied or read
# Utils.check_duplicate(upload.dup)
extension = File.extname("#{upload.filename}")
@ -31,16 +35,84 @@ module Handling
end
filename = Utils.generate_filename
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
File.open(file_path, "w") do |file|
IO.copy(upload.body, file)
File.open(file_path, "w") do |output|
IO.copy(upload.body, output)
end
original_filename = upload.filename
uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
checksum = Utils.hash_file(file_path)
end
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
# X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
# proxy configuration.
ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first
json = JSON.build do |j|
j.object do
j.field "link", "#{protocol}://#{host}/#{filename}"
j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
j.field "id", filename
j.field "ext", extension
j.field "name", original_filename
j.field "checksum", checksum
if CONFIG.delete_key_length > 0
delete_key = Random.base58(CONFIG.delete_key_length)
j.field "deleteKey", delete_key
j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
end
end
end
begin
spawn { Utils.generate_thumbnail(filename, extension) }
rescue ex
LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
end
begin
# Insert SQL data just before returning the upload information
SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
rescue ex
LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
error500("An error ocurred when trying to insert the data into the DB")
end
return json
end
# The most unoptimized and unstable feature lol
def upload_url(env)
env.response.content_type = "application/json"
extension = ""
filename = Utils.generate_filename
original_filename = ""
uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
checksum = ""
ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first
delete_key = nil
# X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
# proxy configuration.
if !env.params.body.nil? || env.params.body["url"].empty?
url = env.params.body["url"]
extension = File.extname(URI.parse(url).path)
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
File.open(file_path, "w") do |output|
begin
HTTP::Client.get(url) do |res|
IO.copy(res.body_io, output)
end
rescue ex
LOGGER.debug "Failed to download file '#{url}': #{ex.message}"
error403("Failed to download file '#{url}'")
end
end
if extension.empty?
extension = Utils.detect_extension(file_path)
File.rename(file_path, file_path + extension)
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
end
# TODO: Benchmark this:
# original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last
original_filename = url.split("/").last
checksum = Utils.hash_file(file_path)
if !filename.empty?
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
@ -60,7 +132,6 @@ module Handling
end
end
begin
LOGGER.debug "Generating thumbnail in background"
spawn { Utils.generate_thumbnail(filename, extension) }
rescue ex
LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
@ -73,29 +144,28 @@ module Handling
LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
error500("An error ocurred when trying to insert the data into the DB")
end
return json
else
LOGGER.debug "No file provided by the user"
error403("No file provided")
end
else
end
error403("Data malformed")
end
def retrieve_file(env)
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
begin
fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum
fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail
FROM #{CONFIG.db_table_name}
WHERE filename = ?",
env.params.url["filename"],
as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String})[0]
env.params.url["filename"].split(".").first,
as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0]
headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"})
headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"})
headers(env, {"ETag" => "#{fileinfo[:checksum]}"})
if env.request.headers.try &.["User-Agent"].includes?("chatterino-api-cache/") || env.request.headers.try &.["User-Agent"].includes?("FFZBot/")
if env.request.headers.try &.["User-Agent"].includes?("chatterino-api-cache/") || env.request.headers.try &.["User-Agent"].includes?("FFZBot/") || env.request.headers.try &.["User-Agent"].includes?("Twitterbot/")
env.response.content_type = "text/html"
return %(
<!DOCTYPE html>
@ -103,7 +173,10 @@ module Handling
<head>
<meta charset="UTF-8">
<meta property="og:title" content="#{fileinfo[:ofilename]}">
<meta property="og:image" content="#{protocol}://#{host}#{CONFIG.thumbnails.split(".")[1]}/#{fileinfo[:filename]}.jpg">
<meta property="og:url" content="#{protocol}://#{host}/#{fileinfo[:filename]}">
#{if fileinfo[:thumbnail]
%(<meta property="og:image" content="#{protocol}://#{host}/thumbnail/#{fileinfo[:filename]}.jpg">)
end}
</head>
</html>
)
@ -117,16 +190,6 @@ module Handling
def retrieve_thumbnail(env)
begin
# fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum
# FROM #{CONFIG.db_table_name}
# WHERE filename = ?",
# env.params.url["filename"],
# as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String})[0]
# headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"})
# headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"})
# headers(env, {"ETag" => "#{fileinfo[:checksum]}"})
send_file env, "#{CONFIG.thumbnails}/#{env.params.url["thumbnail"]}"
rescue ex
LOGGER.debug "Thumbnail '#{env.params.url["thumbnail"]}' does not exist: #{ex.message}"
@ -143,6 +206,8 @@ module Handling
json.object do
json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.db_table_name}", as: Int32
json.field "maxUploadSize", CONFIG.size_limit
json.field "thumbnailGeneration", CONFIG.generate_thumbnails
json.field "filenameLength", CONFIG.filename_length
end
end
end
@ -157,13 +222,19 @@ module Handling
def delete_file(env)
if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.db_table_name} WHERE delete_key = ?)", env.params.query["key"], as: Bool
begin
fileinfo = SQL.query_all("SELECT filename, extension
fileinfo = SQL.query_all("SELECT filename, extension, thumbnail
FROM #{CONFIG.db_table_name}
WHERE delete_key = ?",
env.params.query["key"],
as: {filename: String, extension: String})[0]
as: {filename: String, extension: String, thumbnail: String | Nil})[0]
# Delete file
File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}")
if fileinfo[:thumbnail]
# Delete thumbnail
File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}")
end
# Delete entry from db
SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"]
LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}"
msg("File '#{fileinfo[:filename]}' deleted successfully")
@ -176,4 +247,22 @@ module Handling
error401("Delete key '#{env.params.query["key"]}' does not exist. No files were deleted")
end
end
def sharex_config(env)
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
env.response.content_type = "application/json"
env.response.headers["Content-Disposition"] = "attachment; filename=\"#{host}.sxcu\""
return %({
"Version": "14.0.1",
"DestinationType": "ImageUploader, FileUploader",
"RequestMethod": "POST",
"RequestURL": "#{protocol}://#{host}/upload",
"Body": "MultipartFormData",
"FileFormName": "file",
"URL": "{json:link}",
"DeletionURL": "{json:deleteLink}",
"ErrorMessage": "{json:error}"
})
end
end

12
src/routing.cr
View file

@ -16,11 +16,15 @@ module Routing
Handling.upload(env)
end
post "/api/uploadurl" do |env|
Handling.upload_url(env)
end
get "/:filename" do |env|
Handling.retrieve_file(env)
end
get "/thumbnails/:thumbnail" do |env|
get "/thumbnail/:thumbnail" do |env|
Handling.retrieve_thumbnail(env)
end
@ -28,8 +32,12 @@ module Routing
Handling.delete_file(env)
end
get "/stats" do |env|
get "/api/stats" do |env|
Handling.stats(env)
end
get "/sharex.sxcu" do |env|
Handling.sharex_config(env)
end
end
end

63
src/utils.cr
View file

@ -26,6 +26,20 @@ module Utils
end
end
def create_thumbnails_dir
if !CONFIG.thumbnails
if !Dir.exists?("#{CONFIG.thumbnails}")
LOGGER.info "Creating thumbnaisl folder under '#{CONFIG.thumbnails}'"
begin
Dir.mkdir("#{CONFIG.thumbnails}")
rescue ex
LOGGER.fatal "#{ex.message}"
exit(1)
end
end
end
end
def check_old_files
LOGGER.info "Deleting old files"
dir = Dir.new("#{CONFIG.files}")
@ -47,6 +61,17 @@ module Utils
dir.close
end
def check_dependencies
dependencies = ["ffmpeg"]
dependencies.each do |dep|
next if !CONFIG.generate_thumbnails
if !Process.find_executable(dep)
LOGGER.fatal("'#{dep}' was not found")
exit(1)
end
end
end
# TODO:
# def check_duplicate(upload)
# file_checksum = SQL.query_all("SELECT checksum FROM #{CONFIG.db_table_name} WHERE original_filename = ?", upload.filename, as:String).try &.[0]?
@ -83,22 +108,28 @@ module Utils
end
end
# TODO: Thumbnail generation for videos. Done but error checking IS NOT DONE
def generate_thumbnail(filename, extension)
Process.run("ffmpeg",
# Disable generation if false
return if !CONFIG.generate_thumbnails
LOGGER.debug "Generating thumbnail for #{filename + extension} in background"
process = Process.run("ffmpeg",
[
"-hide_banner",
"-i",
"#{CONFIG.files}/#{filename+extension}",
"#{CONFIG.files}/#{filename + extension}",
"-movflags", "faststart",
"-f", "mjpeg",
"-q:v", "2",
"-vf", "scale='min(350,iw)':'min(350,ih)':force_original_aspect_ratio=decrease, thumbnail=100",
"-frames:v", "1",
"-update", "1",
"#{CONFIG.thumbnails}/#{filename}.jpg"
"#{CONFIG.thumbnails}/#{filename}.jpg",
])
SQL.exec "UPDATE #{CONFIG.db_table_name} SET thumbnail = ? WHERE filename = ?", filename+".jpg", filename
if process.normal_exit?
LOGGER.debug "Thumbnail for #{filename + extension} generated successfully"
SQL.exec "UPDATE #{CONFIG.db_table_name} SET thumbnail = ? WHERE filename = ?", filename + ".jpg", filename
else
end
end
# Delete socket if the server has not been previously cleaned by the server (Due to unclean exits, crashes, etc.)
@ -113,4 +144,26 @@ module Utils
end
end
end
def detect_extension(file) : String
magic_bytes = {
".png" => "89504e470d0a1a0a",
".jpg" => "ffd8ff",
".webm" => "1a45dfa3",
".mp4" => "66747970",
".gif" => "474946383",
".7z" => "377abcaf271c",
".gz" => "1f8b",
}
file = File.open(file)
slice = Bytes.new(8)
hex = IO::Hexdump.new(file)
hex.read(slice)
magic_bytes.each do |ext, mb|
if slice.hexstring.includes?(mb)
return ext
end
end
""
end
end