0.8.1: Option to disable thumbnail generation, OpenGraph implementation

for clients that support it, dynamic ShareX configuration and more.
This commit is contained in:
Fijxu 2024-08-10 00:58:31 -04:00
parent 2058c600bc
commit 43dc289d3a
Signed by: Fijxu
GPG key ID: 32C1DDF333EDA6A4
8 changed files with 234 additions and 90 deletions

View file

@ -9,7 +9,7 @@ Already replaced lol.
- Temporary file uploads like Uguu
- File deletion link (not available in frontend for now)
- Chatterino and ShareX support
- Video Thumbnails for Chatterino and FrankerFaceZ
- Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, can be disabled.)
- Unix socket support if you don't want to deal with all the TCP overhead
- Automatic protocol detection (HTTPS or HTTP)
- Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. It will depend of your traffic.

View file

@ -1,4 +1,6 @@
files: "./files"
thumbnails: "./thumbnails"
generate_thumbnails: false
db: "./db.sqlite3"
db_table_name: "files"
filename_length: 3

View file

@ -1,11 +0,0 @@
{
"Version": "14.0.1",
"DestinationType": "ImageUploader, FileUploader",
"RequestMethod": "POST",
"RequestURL": "https://ayaya.beauty/upload",
"Body": "MultipartFormData",
"FileFormName": "file",
"URL": "{json:link}",
"DeletionURL": "{json:deleteLink}",
"ErrorMessage": "{json:error}"
}

View file

@ -5,8 +5,10 @@ class Config
property files : String = "./files"
property thumbnails : String = "./thumbnails"
property generate_thumbnails : Bool = false
property db : String = "./db.sqlite3"
property db_table_name : String = "files"
property incremental_filename_length : Bool = true
property filename_length : Int32 = 3
# In MiB
property size_limit : Int16 = 512

View file

@ -28,6 +28,7 @@ CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}
Utils.check_dependencies
Utils.create_db
Utils.create_files_dir
Routing.register_all
@ -39,7 +40,6 @@ Jobs.run
Kemal.config.env = "production" if !ENV.has_key?("KEMAL_ENV")
{% end %}
# Set permissions to 777 so NGINX can read and write to it (BROKEN)
if !CONFIG.unix_socket.nil?
sleep 1.second
LOGGER.info "Changing socket permissions to 777"
@ -47,6 +47,7 @@ if !CONFIG.unix_socket.nil?
File.chmod("#{CONFIG.unix_socket}", File::Permissions::All)
rescue ex
LOGGER.fatal "#{ex.message}"
exit(1)
end
end

View file

@ -1,4 +1,5 @@
require "./http-errors"
require "http/client"
module Handling
extend self
@ -22,7 +23,10 @@ module Handling
delete_key = nil
# TODO: Return the file that matches a checksum inside the database
HTTP::FormData.parse(env.request) do |upload|
next if upload.filename.nil? || upload.filename.to_s.empty?
if upload.filename.nil? || upload.filename.to_s.empty?
LOGGER.debug "No file provided by the user"
error403("No file provided")
end
# TODO: upload.body is emptied when is copied or read
# Utils.check_duplicate(upload.dup)
extension = File.extname("#{upload.filename}")
@ -31,71 +35,137 @@ module Handling
end
filename = Utils.generate_filename
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
File.open(file_path, "w") do |file|
IO.copy(upload.body, file)
File.open(file_path, "w") do |output|
IO.copy(upload.body, output)
end
original_filename = upload.filename
uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
checksum = Utils.hash_file(file_path)
# X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
# proxy configuration.
ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first
end
if !filename.empty?
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
json = JSON.build do |j|
j.object do
j.field "link", "#{protocol}://#{host}/#{filename}"
j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
j.field "id", filename
j.field "ext", extension
j.field "name", original_filename
j.field "checksum", checksum
if CONFIG.delete_key_length > 0
delete_key = Random.base58(CONFIG.delete_key_length)
j.field "deleteKey", delete_key
j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
end
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
# X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
# proxy configuration.
ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first
json = JSON.build do |j|
j.object do
j.field "link", "#{protocol}://#{host}/#{filename}"
j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
j.field "id", filename
j.field "ext", extension
j.field "name", original_filename
j.field "checksum", checksum
if CONFIG.delete_key_length > 0
delete_key = Random.base58(CONFIG.delete_key_length)
j.field "deleteKey", delete_key
j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
end
end
begin
LOGGER.debug "Generating thumbnail in background"
spawn { Utils.generate_thumbnail(filename, extension) }
rescue ex
LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
end
begin
# Insert SQL data just before returning the upload information
SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
rescue ex
LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
error500("An error ocurred when trying to insert the data into the DB")
end
return json
else
LOGGER.debug "No file provided by the user"
error403("No file provided")
end
begin
spawn { Utils.generate_thumbnail(filename, extension) }
rescue ex
LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
end
begin
# Insert SQL data just before returning the upload information
SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
rescue ex
LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
error500("An error ocurred when trying to insert the data into the DB")
end
return json
end
# The most unoptimized and unstable feature lol
def upload_url(env)
env.response.content_type = "application/json"
extension = ""
filename = Utils.generate_filename
original_filename = ""
uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
checksum = ""
ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first
delete_key = nil
# X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
# proxy configuration.
if !env.params.body.nil? || env.params.body["url"].empty?
url = env.params.body["url"]
extension = File.extname(URI.parse(url).path)
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
File.open(file_path, "w") do |output|
begin
HTTP::Client.get(url) do |res|
IO.copy(res.body_io, output)
end
rescue ex
LOGGER.debug "Failed to download file '#{url}': #{ex.message}"
error403("Failed to download file '#{url}'")
end
end
if extension.empty?
extension = Utils.detect_extension(file_path)
File.rename(file_path, file_path + extension)
file_path = ::File.join ["#{CONFIG.files}", filename + extension]
end
# TODO: Benchmark this:
# original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last
original_filename = url.split("/").last
checksum = Utils.hash_file(file_path)
if !filename.empty?
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
json = JSON.build do |j|
j.object do
j.field "link", "#{protocol}://#{host}/#{filename}"
j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
j.field "id", filename
j.field "ext", extension
j.field "name", original_filename
j.field "checksum", checksum
if CONFIG.delete_key_length > 0
delete_key = Random.base58(CONFIG.delete_key_length)
j.field "deleteKey", delete_key
j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
end
end
end
begin
spawn { Utils.generate_thumbnail(filename, extension) }
rescue ex
LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
end
begin
# Insert SQL data just before returning the upload information
SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
rescue ex
LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
error500("An error ocurred when trying to insert the data into the DB")
end
return json
end
else
end
error403("Data malformed")
end
def retrieve_file(env)
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
begin
fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum
fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail
FROM #{CONFIG.db_table_name}
WHERE filename = ?",
env.params.url["filename"],
as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String})[0]
env.params.url["filename"].split(".").first,
as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0]
headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"})
headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"})
headers(env, {"ETag" => "#{fileinfo[:checksum]}"})
if env.request.headers.try &.["User-Agent"].includes?("chatterino-api-cache/") || env.request.headers.try &.["User-Agent"].includes?("FFZBot/")
if env.request.headers.try &.["User-Agent"].includes?("chatterino-api-cache/") || env.request.headers.try &.["User-Agent"].includes?("FFZBot/") || env.request.headers.try &.["User-Agent"].includes?("Twitterbot/")
env.response.content_type = "text/html"
return %(
<!DOCTYPE html>
@ -103,7 +173,10 @@ module Handling
<head>
<meta charset="UTF-8">
<meta property="og:title" content="#{fileinfo[:ofilename]}">
<meta property="og:image" content="#{protocol}://#{host}#{CONFIG.thumbnails.split(".")[1]}/#{fileinfo[:filename]}.jpg">
<meta property="og:url" content="#{protocol}://#{host}/#{fileinfo[:filename]}">
#{if fileinfo[:thumbnail]
%(<meta property="og:image" content="#{protocol}://#{host}/thumbnail/#{fileinfo[:filename]}.jpg">)
end}
</head>
</html>
)
@ -117,16 +190,6 @@ module Handling
def retrieve_thumbnail(env)
begin
# fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum
# FROM #{CONFIG.db_table_name}
# WHERE filename = ?",
# env.params.url["filename"],
# as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String})[0]
# headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"})
# headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"})
# headers(env, {"ETag" => "#{fileinfo[:checksum]}"})
send_file env, "#{CONFIG.thumbnails}/#{env.params.url["thumbnail"]}"
rescue ex
LOGGER.debug "Thumbnail '#{env.params.url["thumbnail"]}' does not exist: #{ex.message}"
@ -143,6 +206,8 @@ module Handling
json.object do
json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.db_table_name}", as: Int32
json.field "maxUploadSize", CONFIG.size_limit
json.field "thumbnailGeneration", CONFIG.generate_thumbnails
json.field "filenameLength", CONFIG.filename_length
end
end
end
@ -157,13 +222,19 @@ module Handling
def delete_file(env)
if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.db_table_name} WHERE delete_key = ?)", env.params.query["key"], as: Bool
begin
fileinfo = SQL.query_all("SELECT filename, extension
fileinfo = SQL.query_all("SELECT filename, extension, thumbnail
FROM #{CONFIG.db_table_name}
WHERE delete_key = ?",
env.params.query["key"],
as: {filename: String, extension: String})[0]
env.params.query["key"],
as: {filename: String, extension: String, thumbnail: String | Nil})[0]
# Delete file
File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}")
if fileinfo[:thumbnail]
# Delete thumbnail
File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}")
end
# Delete entry from db
SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"]
LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}"
msg("File '#{fileinfo[:filename]}' deleted successfully")
@ -176,4 +247,22 @@ module Handling
error401("Delete key '#{env.params.query["key"]}' does not exist. No files were deleted")
end
end
def sharex_config(env)
protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
env.response.content_type = "application/json"
env.response.headers["Content-Disposition"] = "attachment; filename=\"#{host}.sxcu\""
return %({
"Version": "14.0.1",
"DestinationType": "ImageUploader, FileUploader",
"RequestMethod": "POST",
"RequestURL": "#{protocol}://#{host}/upload",
"Body": "MultipartFormData",
"FileFormName": "file",
"URL": "{json:link}",
"DeletionURL": "{json:deleteLink}",
"ErrorMessage": "{json:error}"
})
end
end

View file

@ -16,11 +16,15 @@ module Routing
Handling.upload(env)
end
post "/api/uploadurl" do |env|
Handling.upload_url(env)
end
get "/:filename" do |env|
Handling.retrieve_file(env)
end
get "/thumbnails/:thumbnail" do |env|
get "/thumbnail/:thumbnail" do |env|
Handling.retrieve_thumbnail(env)
end
@ -28,8 +32,12 @@ module Routing
Handling.delete_file(env)
end
get "/stats" do |env|
get "/api/stats" do |env|
Handling.stats(env)
end
get "/sharex.sxcu" do |env|
Handling.sharex_config(env)
end
end
end

View file

@ -26,6 +26,20 @@ module Utils
end
end
def create_thumbnails_dir
if !CONFIG.thumbnails
if !Dir.exists?("#{CONFIG.thumbnails}")
LOGGER.info "Creating thumbnaisl folder under '#{CONFIG.thumbnails}'"
begin
Dir.mkdir("#{CONFIG.thumbnails}")
rescue ex
LOGGER.fatal "#{ex.message}"
exit(1)
end
end
end
end
def check_old_files
LOGGER.info "Deleting old files"
dir = Dir.new("#{CONFIG.files}")
@ -39,7 +53,7 @@ module Utils
File.delete("#{CONFIG.files}/#{file}")
rescue ex
LOGGER.error "#{ex.message}"
end
end
end
end
# Close directory to prevent `Too many open files (File::Error)` error.
@ -47,6 +61,17 @@ module Utils
dir.close
end
def check_dependencies
dependencies = ["ffmpeg"]
dependencies.each do |dep|
next if !CONFIG.generate_thumbnails
if !Process.find_executable(dep)
LOGGER.fatal("'#{dep}' was not found")
exit(1)
end
end
end
# TODO:
# def check_duplicate(upload)
# file_checksum = SQL.query_all("SELECT checksum FROM #{CONFIG.db_table_name} WHERE original_filename = ?", upload.filename, as:String).try &.[0]?
@ -83,22 +108,28 @@ module Utils
end
end
# TODO: Thumbnail generation for videos. Done but error checking IS NOT DONE
def generate_thumbnail(filename, extension)
Process.run("ffmpeg",
# Disable generation if false
return if !CONFIG.generate_thumbnails
LOGGER.debug "Generating thumbnail for #{filename + extension} in background"
process = Process.run("ffmpeg",
[
"-hide_banner",
"-i",
"#{CONFIG.files}/#{filename+extension}",
"-movflags", "faststart",
"-f", "mjpeg",
"-q:v", "2",
"-vf", "scale='min(350,iw)':'min(350,ih)':force_original_aspect_ratio=decrease, thumbnail=100",
"-frames:v", "1",
"-update", "1",
"#{CONFIG.thumbnails}/#{filename}.jpg"
"-hide_banner",
"-i",
"#{CONFIG.files}/#{filename + extension}",
"-movflags", "faststart",
"-f", "mjpeg",
"-q:v", "2",
"-vf", "scale='min(350,iw)':'min(350,ih)':force_original_aspect_ratio=decrease, thumbnail=100",
"-frames:v", "1",
"-update", "1",
"#{CONFIG.thumbnails}/#{filename}.jpg",
])
SQL.exec "UPDATE #{CONFIG.db_table_name} SET thumbnail = ? WHERE filename = ?", filename+".jpg", filename
if process.normal_exit?
LOGGER.debug "Thumbnail for #{filename + extension} generated successfully"
SQL.exec "UPDATE #{CONFIG.db_table_name} SET thumbnail = ? WHERE filename = ?", filename + ".jpg", filename
else
end
end
# Delete socket if the server has not been previously cleaned by the server (Due to unclean exits, crashes, etc.)
@ -113,4 +144,26 @@ module Utils
end
end
end
def detect_extension(file) : String
magic_bytes = {
".png" => "89504e470d0a1a0a",
".jpg" => "ffd8ff",
".webm" => "1a45dfa3",
".mp4" => "66747970",
".gif" => "474946383",
".7z" => "377abcaf271c",
".gz" => "1f8b",
}
file = File.open(file)
slice = Bytes.new(8)
hex = IO::Hexdump.new(file)
hex.read(slice)
magic_bytes.each do |ext, mb|
if slice.hexstring.includes?(mb)
return ext
end
end
""
end
end