diff --git a/README.md b/README.md index 0210b84..d91cc57 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Already replaced lol. - Temporary file uploads like Uguu - File deletion link (not available in frontend for now) - Chatterino and ShareX support -- Video Thumbnails for Chatterino and FrankerFaceZ +- Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, can be disabled.) - Unix socket support if you don't want to deal with all the TCP overhead - Automatic protocol detection (HTTPS or HTTP) - Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. It will depend of your traffic. diff --git a/config/config.yml b/config/config.yml index b3735dc..0e0e34b 100644 --- a/config/config.yml +++ b/config/config.yml @@ -1,4 +1,6 @@ files: "./files" +thumbnails: "./thumbnails" +generate_thumbnails: false db: "./db.sqlite3" db_table_name: "files" filename_length: 3 diff --git a/public/sharex.sxcu b/public/sharex.sxcu deleted file mode 100644 index 2f14991..0000000 --- a/public/sharex.sxcu +++ /dev/null @@ -1,11 +0,0 @@ -{ - "Version": "14.0.1", - "DestinationType": "ImageUploader, FileUploader", - "RequestMethod": "POST", - "RequestURL": "https://ayaya.beauty/upload", - "Body": "MultipartFormData", - "FileFormName": "file", - "URL": "{json:link}", - "DeletionURL": "{json:deleteLink}", - "ErrorMessage": "{json:error}" -} \ No newline at end of file diff --git a/src/config.cr b/src/config.cr index f1ed2e9..062909f 100644 --- a/src/config.cr +++ b/src/config.cr @@ -5,8 +5,10 @@ class Config property files : String = "./files" property thumbnails : String = "./thumbnails" + property generate_thumbnails : Bool = false property db : String = "./db.sqlite3" property db_table_name : String = "files" + property incremental_filename_length : Bool = true property filename_length : Int32 = 3 # In MiB property size_limit : Int16 = 512 diff --git a/src/file-uploader.cr b/src/file-uploader.cr index 004596e..e8ddda7 100644 --- a/src/file-uploader.cr +++ b/src/file-uploader.cr @@ -28,6 +28,7 @@ CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }} CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }} CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }} +Utils.check_dependencies Utils.create_db Utils.create_files_dir Routing.register_all @@ -39,7 +40,6 @@ Jobs.run Kemal.config.env = "production" if !ENV.has_key?("KEMAL_ENV") {% end %} -# Set permissions to 777 so NGINX can read and write to it (BROKEN) if !CONFIG.unix_socket.nil? sleep 1.second LOGGER.info "Changing socket permissions to 777" @@ -47,6 +47,7 @@ if !CONFIG.unix_socket.nil? File.chmod("#{CONFIG.unix_socket}", File::Permissions::All) rescue ex LOGGER.fatal "#{ex.message}" + exit(1) end end diff --git a/src/handling.cr b/src/handling.cr index 8a8fb13..26eb14c 100644 --- a/src/handling.cr +++ b/src/handling.cr @@ -1,4 +1,5 @@ require "./http-errors" +require "http/client" module Handling extend self @@ -22,7 +23,10 @@ module Handling delete_key = nil # TODO: Return the file that matches a checksum inside the database HTTP::FormData.parse(env.request) do |upload| - next if upload.filename.nil? || upload.filename.to_s.empty? + if upload.filename.nil? || upload.filename.to_s.empty? + LOGGER.debug "No file provided by the user" + error403("No file provided") + end # TODO: upload.body is emptied when is copied or read # Utils.check_duplicate(upload.dup) extension = File.extname("#{upload.filename}") @@ -31,71 +35,137 @@ module Handling end filename = Utils.generate_filename file_path = ::File.join ["#{CONFIG.files}", filename + extension] - File.open(file_path, "w") do |file| - IO.copy(upload.body, file) + File.open(file_path, "w") do |output| + IO.copy(upload.body, output) end original_filename = upload.filename uploaded_at = Time::Format::HTTP_DATE.format(Time.utc) checksum = Utils.hash_file(file_path) - # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse - # proxy configuration. - ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first end - if !filename.empty? - protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" - host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] - json = JSON.build do |j| - j.object do - j.field "link", "#{protocol}://#{host}/#{filename}" - j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}" - j.field "id", filename - j.field "ext", extension - j.field "name", original_filename - j.field "checksum", checksum - if CONFIG.delete_key_length > 0 - delete_key = Random.base58(CONFIG.delete_key_length) - j.field "deleteKey", delete_key - j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}" - end + protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" + host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] + # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse + # proxy configuration. + ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first + json = JSON.build do |j| + j.object do + j.field "link", "#{protocol}://#{host}/#{filename}" + j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}" + j.field "id", filename + j.field "ext", extension + j.field "name", original_filename + j.field "checksum", checksum + if CONFIG.delete_key_length > 0 + delete_key = Random.base58(CONFIG.delete_key_length) + j.field "deleteKey", delete_key + j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}" end end - begin - LOGGER.debug "Generating thumbnail in background" - spawn { Utils.generate_thumbnail(filename, extension) } - rescue ex - LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" - end - begin - # Insert SQL data just before returning the upload information - SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil - rescue ex - LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" - error500("An error ocurred when trying to insert the data into the DB") - end - - return json - else - LOGGER.debug "No file provided by the user" - error403("No file provided") end + begin + spawn { Utils.generate_thumbnail(filename, extension) } + rescue ex + LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" + end + begin + # Insert SQL data just before returning the upload information + SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil + rescue ex + LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" + error500("An error ocurred when trying to insert the data into the DB") + end + return json + end + + # The most unoptimized and unstable feature lol + def upload_url(env) + env.response.content_type = "application/json" + extension = "" + filename = Utils.generate_filename + original_filename = "" + uploaded_at = Time::Format::HTTP_DATE.format(Time.utc) + checksum = "" + ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first + delete_key = nil + # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse + # proxy configuration. + if !env.params.body.nil? || env.params.body["url"].empty? + url = env.params.body["url"] + extension = File.extname(URI.parse(url).path) + file_path = ::File.join ["#{CONFIG.files}", filename + extension] + File.open(file_path, "w") do |output| + begin + HTTP::Client.get(url) do |res| + IO.copy(res.body_io, output) + end + rescue ex + LOGGER.debug "Failed to download file '#{url}': #{ex.message}" + error403("Failed to download file '#{url}'") + end + end + if extension.empty? + extension = Utils.detect_extension(file_path) + File.rename(file_path, file_path + extension) + file_path = ::File.join ["#{CONFIG.files}", filename + extension] + end + # TODO: Benchmark this: + # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last + original_filename = url.split("/").last + checksum = Utils.hash_file(file_path) + if !filename.empty? + protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" + host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] + json = JSON.build do |j| + j.object do + j.field "link", "#{protocol}://#{host}/#{filename}" + j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}" + j.field "id", filename + j.field "ext", extension + j.field "name", original_filename + j.field "checksum", checksum + if CONFIG.delete_key_length > 0 + delete_key = Random.base58(CONFIG.delete_key_length) + j.field "deleteKey", delete_key + j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}" + end + end + end + begin + spawn { Utils.generate_thumbnail(filename, extension) } + rescue ex + LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" + end + begin + # Insert SQL data just before returning the upload information + SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil + rescue ex + LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" + error500("An error ocurred when trying to insert the data into the DB") + end + return json + end + else + end + error403("Data malformed") end def retrieve_file(env) - protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" - host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] + protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" + host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] begin - fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum + fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail FROM #{CONFIG.db_table_name} WHERE filename = ?", - env.params.url["filename"], - as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String})[0] + env.params.url["filename"].split(".").first, + as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0] headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"}) headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"}) headers(env, {"ETag" => "#{fileinfo[:checksum]}"}) - if env.request.headers.try &.["User-Agent"].includes?("chatterino-api-cache/") || env.request.headers.try &.["User-Agent"].includes?("FFZBot/") + if env.request.headers.try &.["User-Agent"].includes?("chatterino-api-cache/") || env.request.headers.try &.["User-Agent"].includes?("FFZBot/") || env.request.headers.try &.["User-Agent"].includes?("Twitterbot/") env.response.content_type = "text/html" return %( @@ -103,7 +173,10 @@ module Handling - + + #{if fileinfo[:thumbnail] + %() + end} ) @@ -117,16 +190,6 @@ module Handling def retrieve_thumbnail(env) begin - # fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum - # FROM #{CONFIG.db_table_name} - # WHERE filename = ?", - # env.params.url["filename"], - # as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String})[0] - - # headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"}) - # headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"}) - # headers(env, {"ETag" => "#{fileinfo[:checksum]}"}) - send_file env, "#{CONFIG.thumbnails}/#{env.params.url["thumbnail"]}" rescue ex LOGGER.debug "Thumbnail '#{env.params.url["thumbnail"]}' does not exist: #{ex.message}" @@ -143,6 +206,8 @@ module Handling json.object do json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.db_table_name}", as: Int32 json.field "maxUploadSize", CONFIG.size_limit + json.field "thumbnailGeneration", CONFIG.generate_thumbnails + json.field "filenameLength", CONFIG.filename_length end end end @@ -157,13 +222,19 @@ module Handling def delete_file(env) if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.db_table_name} WHERE delete_key = ?)", env.params.query["key"], as: Bool begin - fileinfo = SQL.query_all("SELECT filename, extension + fileinfo = SQL.query_all("SELECT filename, extension, thumbnail FROM #{CONFIG.db_table_name} WHERE delete_key = ?", - env.params.query["key"], - as: {filename: String, extension: String})[0] + env.params.query["key"], + as: {filename: String, extension: String, thumbnail: String | Nil})[0] + # Delete file File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}") + if fileinfo[:thumbnail] + # Delete thumbnail + File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") + end + # Delete entry from db SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"] LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" msg("File '#{fileinfo[:filename]}' deleted successfully") @@ -176,4 +247,22 @@ module Handling error401("Delete key '#{env.params.query["key"]}' does not exist. No files were deleted") end end + + def sharex_config(env) + protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" + host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] + env.response.content_type = "application/json" + env.response.headers["Content-Disposition"] = "attachment; filename=\"#{host}.sxcu\"" + return %({ + "Version": "14.0.1", + "DestinationType": "ImageUploader, FileUploader", + "RequestMethod": "POST", + "RequestURL": "#{protocol}://#{host}/upload", + "Body": "MultipartFormData", + "FileFormName": "file", + "URL": "{json:link}", + "DeletionURL": "{json:deleteLink}", + "ErrorMessage": "{json:error}" +}) + end end diff --git a/src/routing.cr b/src/routing.cr index 4b44464..c9bd297 100644 --- a/src/routing.cr +++ b/src/routing.cr @@ -16,11 +16,15 @@ module Routing Handling.upload(env) end + post "/api/uploadurl" do |env| + Handling.upload_url(env) + end + get "/:filename" do |env| Handling.retrieve_file(env) end - get "/thumbnails/:thumbnail" do |env| + get "/thumbnail/:thumbnail" do |env| Handling.retrieve_thumbnail(env) end @@ -28,8 +32,12 @@ module Routing Handling.delete_file(env) end - get "/stats" do |env| + get "/api/stats" do |env| Handling.stats(env) end + + get "/sharex.sxcu" do |env| + Handling.sharex_config(env) + end end end diff --git a/src/utils.cr b/src/utils.cr index 8d33ee9..33e591d 100644 --- a/src/utils.cr +++ b/src/utils.cr @@ -26,6 +26,20 @@ module Utils end end + def create_thumbnails_dir + if !CONFIG.thumbnails + if !Dir.exists?("#{CONFIG.thumbnails}") + LOGGER.info "Creating thumbnaisl folder under '#{CONFIG.thumbnails}'" + begin + Dir.mkdir("#{CONFIG.thumbnails}") + rescue ex + LOGGER.fatal "#{ex.message}" + exit(1) + end + end + end + end + def check_old_files LOGGER.info "Deleting old files" dir = Dir.new("#{CONFIG.files}") @@ -39,7 +53,7 @@ module Utils File.delete("#{CONFIG.files}/#{file}") rescue ex LOGGER.error "#{ex.message}" - end + end end end # Close directory to prevent `Too many open files (File::Error)` error. @@ -47,6 +61,17 @@ module Utils dir.close end + def check_dependencies + dependencies = ["ffmpeg"] + dependencies.each do |dep| + next if !CONFIG.generate_thumbnails + if !Process.find_executable(dep) + LOGGER.fatal("'#{dep}' was not found") + exit(1) + end + end + end + # TODO: # def check_duplicate(upload) # file_checksum = SQL.query_all("SELECT checksum FROM #{CONFIG.db_table_name} WHERE original_filename = ?", upload.filename, as:String).try &.[0]? @@ -83,22 +108,28 @@ module Utils end end - # TODO: Thumbnail generation for videos. Done but error checking IS NOT DONE def generate_thumbnail(filename, extension) - Process.run("ffmpeg", + # Disable generation if false + return if !CONFIG.generate_thumbnails + LOGGER.debug "Generating thumbnail for #{filename + extension} in background" + process = Process.run("ffmpeg", [ - "-hide_banner", - "-i", - "#{CONFIG.files}/#{filename+extension}", - "-movflags", "faststart", - "-f", "mjpeg", - "-q:v", "2", - "-vf", "scale='min(350,iw)':'min(350,ih)':force_original_aspect_ratio=decrease, thumbnail=100", - "-frames:v", "1", - "-update", "1", - "#{CONFIG.thumbnails}/#{filename}.jpg" + "-hide_banner", + "-i", + "#{CONFIG.files}/#{filename + extension}", + "-movflags", "faststart", + "-f", "mjpeg", + "-q:v", "2", + "-vf", "scale='min(350,iw)':'min(350,ih)':force_original_aspect_ratio=decrease, thumbnail=100", + "-frames:v", "1", + "-update", "1", + "#{CONFIG.thumbnails}/#{filename}.jpg", ]) - SQL.exec "UPDATE #{CONFIG.db_table_name} SET thumbnail = ? WHERE filename = ?", filename+".jpg", filename + if process.normal_exit? + LOGGER.debug "Thumbnail for #{filename + extension} generated successfully" + SQL.exec "UPDATE #{CONFIG.db_table_name} SET thumbnail = ? WHERE filename = ?", filename + ".jpg", filename + else + end end # Delete socket if the server has not been previously cleaned by the server (Due to unclean exits, crashes, etc.) @@ -113,4 +144,26 @@ module Utils end end end + + def detect_extension(file) : String + magic_bytes = { + ".png" => "89504e470d0a1a0a", + ".jpg" => "ffd8ff", + ".webm" => "1a45dfa3", + ".mp4" => "66747970", + ".gif" => "474946383", + ".7z" => "377abcaf271c", + ".gz" => "1f8b", + } + file = File.open(file) + slice = Bytes.new(8) + hex = IO::Hexdump.new(file) + hex.read(slice) + magic_bytes.each do |ext, mb| + if slice.hexstring.includes?(mb) + return ext + end + end + "" + end end