From 3173a36a9928685201988551ada7cfcbffc651f7 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Sat, 10 Aug 2024 03:09:38 -0400 Subject: [PATCH] 0.8.2: Admin API, confiugurable OpenGraph user-agents. --- README.md | 1 + config/config.yml | 8 ++ src/config.cr | 3 + src/file-uploader.cr | 3 +- src/handling/admin.cr | 49 +++++++ src/handling/handling.cr | 270 +++++++++++++++++++++++++++++++++++++++ src/routing.cr | 6 + src/utils.cr | 21 ++- src/views/index.ecr | 2 +- 9 files changed, 360 insertions(+), 3 deletions(-) create mode 100644 src/handling/admin.cr create mode 100644 src/handling/handling.cr diff --git a/README.md b/README.md index d91cc57..96d6c13 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Already replaced lol. - File deletion link (not available in frontend for now) - Chatterino and ShareX support - Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, can be disabled.) +- [Small Admin API](./src/handling/admin.cr) that allows you to delete files. (Needs to be enabled in the configuration) - Unix socket support if you don't want to deal with all the TCP overhead - Automatic protocol detection (HTTPS or HTTP) - Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. It will depend of your traffic. diff --git a/config/config.yml b/config/config.yml index 0e0e34b..4ec2611 100644 --- a/config/config.yml +++ b/config/config.yml @@ -3,6 +3,8 @@ thumbnails: "./thumbnails" generate_thumbnails: false db: "./db.sqlite3" db_table_name: "files" +adminEnabled: false +adminApiKey: "asd" filename_length: 3 # In MiB size_limit: 512 @@ -20,3 +22,9 @@ log_level: "debug" blocked_extensions: - "exe" + +# List of useragents that use OpenGraph to gather file information +opengraph_useragents: + - "chatterino-api-cache/" + - "FFZBot/" + - "Twitterbot/" diff --git a/src/config.cr b/src/config.cr index 062909f..674e556 100644 --- a/src/config.cr +++ b/src/config.cr @@ -8,6 +8,8 @@ class Config property generate_thumbnails : Bool = false property db : String = "./db.sqlite3" property db_table_name : String = "files" + property adminEnabled : Bool = false + property adminApiKey : String = "" property incremental_filename_length : Bool = true property filename_length : Int32 = 3 # In MiB @@ -20,6 +22,7 @@ class Config property delete_key_length : Int32 = 4 # Blocked extensions that are not allowed to be uploaded to the server property blocked_extensions : Array(String) = [] of String + property opengraph_useragents : Array(String) = [] of String property siteInfo : String = "xd" property siteWarning : String? = "" property log_level : LogLevel = LogLevel::Info diff --git a/src/file-uploader.cr b/src/file-uploader.cr index e8ddda7..7504f59 100644 --- a/src/file-uploader.cr +++ b/src/file-uploader.cr @@ -8,7 +8,7 @@ require "digest" require "./logger" require "./routing" require "./utils" -require "./handling" +require "./handling/**" require "./config" require "./jobs" require "./lib/**" @@ -27,6 +27,7 @@ SQL = DB.open("sqlite3://#{CONFIG.db}") CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }} CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }} CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }} +CURRENT_TAG = {{ "#{`git describe --long --abbrev=7 --tags | sed 's/([^-]*)-g.*/r\1/;s/-/./g'`.strip}" }} Utils.check_dependencies Utils.create_db diff --git a/src/handling/admin.cr b/src/handling/admin.cr new file mode 100644 index 0000000..c83933b --- /dev/null +++ b/src/handling/admin.cr @@ -0,0 +1,49 @@ +require "../http-errors" + +module Handling::Admin + extend self + + def delete_file(env) + if env.request.headers.try &.["X-Api-Key"]? != CONFIG.adminApiKey || nil + error401 "Wrong API Key" + end + files = env.params.json["files"].as((Array(JSON::Any))) + successfull_files = [] of String + failed_files = [] of String + files.each do |file| + file = file.to_s + begin + fileinfo = SQL.query_one("SELECT filename, extension, thumbnail + FROM #{CONFIG.db_table_name} + WHERE filename = ?", + file, + as: {filename: String, extension: String, thumbnail: String | Nil}) + + # Delete file + File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}") + if fileinfo[:thumbnail] + # Delete thumbnail + File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") + end + # Delete entry from db + SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE filename = ?", file + LOGGER.debug "File '#{fileinfo[:filename]}' was deleted" + successfull_files << file + rescue ex : DB::NoResultsError + LOGGER.error("File '#{file}' doesn't exist: #{ex.message}") + failed_files << file + rescue ex + LOGGER.error "Unknown error: #{ex.message}" + error500 "Unknown error: #{ex.message}" + end + end + json = JSON.build do |j| + j.object do + j.field "successfull", successfull_files.size + j.field "failed", failed_files.size + j.field "successfullFiles", successfull_files + j.field "failedFiles", failed_files + end + end + end +end diff --git a/src/handling/handling.cr b/src/handling/handling.cr new file mode 100644 index 0000000..1af6c71 --- /dev/null +++ b/src/handling/handling.cr @@ -0,0 +1,270 @@ +require "../http-errors" +require "http/client" + +module Handling + extend self + + def upload(env) + env.response.content_type = "application/json" + # You can modify this if you want to allow files smaller than 1MiB. + # This is generally a good way to check the filesize but there is a better way to do it + # which is inspecting the file directly (If I'm not wrong). + if CONFIG.size_limit > 0 + if env.request.headers["Content-Length"].to_i > 1048576*CONFIG.size_limit + error413("File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB") + end + end + filename = "" + extension = "" + original_filename = "" + uploaded_at = "" + checksum = "" + ip_address = "" + delete_key = nil + # TODO: Return the file that matches a checksum inside the database + HTTP::FormData.parse(env.request) do |upload| + if upload.filename.nil? || upload.filename.to_s.empty? + LOGGER.debug "No file provided by the user" + error403("No file provided") + end + # TODO: upload.body is emptied when is copied or read + # Utils.check_duplicate(upload.dup) + extension = File.extname("#{upload.filename}") + if CONFIG.blocked_extensions.includes?(extension.split(".")[1]) + error401("Extension '#{extension}' is not allowed") + end + filename = Utils.generate_filename + file_path = ::File.join ["#{CONFIG.files}", filename + extension] + File.open(file_path, "w") do |output| + IO.copy(upload.body, output) + end + original_filename = upload.filename + uploaded_at = Time::Format::HTTP_DATE.format(Time.utc) + checksum = Utils.hash_file(file_path) + end + protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" + host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] + # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse + # proxy configuration. + ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first + json = JSON.build do |j| + j.object do + j.field "link", "#{protocol}://#{host}/#{filename}" + j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}" + j.field "id", filename + j.field "ext", extension + j.field "name", original_filename + j.field "checksum", checksum + if CONFIG.delete_key_length > 0 + delete_key = Random.base58(CONFIG.delete_key_length) + j.field "deleteKey", delete_key + j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}" + end + end + end + begin + spawn { Utils.generate_thumbnail(filename, extension) } + rescue ex + LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" + end + begin + # Insert SQL data just before returning the upload information + SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil + rescue ex + LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" + error500("An error ocurred when trying to insert the data into the DB") + end + return json + end + + # The most unoptimized and unstable feature lol + def upload_url(env) + env.response.content_type = "application/json" + extension = "" + filename = Utils.generate_filename + original_filename = "" + uploaded_at = Time::Format::HTTP_DATE.format(Time.utc) + checksum = "" + ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first + delete_key = nil + # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse + # proxy configuration. + if !env.params.body.nil? || env.params.body["url"].empty? + url = env.params.body["url"] + extension = File.extname(URI.parse(url).path) + file_path = ::File.join ["#{CONFIG.files}", filename + extension] + File.open(file_path, "w") do |output| + begin + HTTP::Client.get(url) do |res| + IO.copy(res.body_io, output) + end + rescue ex + LOGGER.debug "Failed to download file '#{url}': #{ex.message}" + error403("Failed to download file '#{url}'") + end + end + if extension.empty? + extension = Utils.detect_extension(file_path) + File.rename(file_path, file_path + extension) + file_path = ::File.join ["#{CONFIG.files}", filename + extension] + end + # TODO: Benchmark this: + # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last + original_filename = url.split("/").last + checksum = Utils.hash_file(file_path) + if !filename.empty? + protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" + host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] + json = JSON.build do |j| + j.object do + j.field "link", "#{protocol}://#{host}/#{filename}" + j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}" + j.field "id", filename + j.field "ext", extension + j.field "name", original_filename + j.field "checksum", checksum + if CONFIG.delete_key_length > 0 + delete_key = Random.base58(CONFIG.delete_key_length) + j.field "deleteKey", delete_key + j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}" + end + end + end + begin + spawn { Utils.generate_thumbnail(filename, extension) } + rescue ex + LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" + end + begin + # Insert SQL data just before returning the upload information + SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil + rescue ex + LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" + error500("An error ocurred when trying to insert the data into the DB") + end + return json + end + else + end + error403("Data malformed") + end + + def retrieve_file(env) + protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" + host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] + begin + fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail + FROM #{CONFIG.db_table_name} + WHERE filename = ?", + env.params.url["filename"].split(".").first, + as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0] + + headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"}) + headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"}) + headers(env, {"ETag" => "#{fileinfo[:checksum]}"}) + + CONFIG.opengraph_useragents.each do |useragent| + if env.request.headers.try &.["User-Agent"].includes?(useragent) + env.response.content_type = "text/html" + return %( + + + + + + + #{if fileinfo[:thumbnail] + %() + end} + + +) + end + end + send_file env, "#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:ext]}" + rescue ex + LOGGER.debug "File '#{env.params.url["filename"]}' does not exist: #{ex.message}" + error403("File '#{env.params.url["filename"]}' does not exist") + end + end + + def retrieve_thumbnail(env) + begin + send_file env, "#{CONFIG.thumbnails}/#{env.params.url["thumbnail"]}" + rescue ex + LOGGER.debug "Thumbnail '#{env.params.url["thumbnail"]}' does not exist: #{ex.message}" + error403("Thumbnail '#{env.params.url["thumbnail"]}' does not exist") + end + end + + def stats(env) + env.response.content_type = "application/json" + begin + json_data = JSON.build do |json| + json.object do + json.field "stats" do + json.object do + json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.db_table_name}", as: Int32 + json.field "maxUploadSize", CONFIG.size_limit + json.field "thumbnailGeneration", CONFIG.generate_thumbnails + json.field "filenameLength", CONFIG.filename_length + end + end + end + end + rescue ex + LOGGER.error "Unknown error: #{ex.message}" + error500("Unknown error") + end + json_data + end + + def delete_file(env) + if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.db_table_name} WHERE delete_key = ?)", env.params.query["key"], as: Bool + begin + fileinfo = SQL.query_all("SELECT filename, extension, thumbnail + FROM #{CONFIG.db_table_name} + WHERE delete_key = ?", + env.params.query["key"], + as: {filename: String, extension: String, thumbnail: String | Nil})[0] + + # Delete file + File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}") + if fileinfo[:thumbnail] + # Delete thumbnail + File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") + end + # Delete entry from db + SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"] + LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" + msg("File '#{fileinfo[:filename]}' deleted successfully") + rescue ex + LOGGER.error("Unknown error: #{ex.message}") + error500("Unknown error") + end + else + LOGGER.debug "Key '#{env.params.query["key"]}' does not exist" + error401("Delete key '#{env.params.query["key"]}' does not exist. No files were deleted") + end + end + + def sharex_config(env) + protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" + host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] + env.response.content_type = "application/json" + env.response.headers["Content-Disposition"] = "attachment; filename=\"#{host}.sxcu\"" + return %({ + "Version": "14.0.1", + "DestinationType": "ImageUploader, FileUploader", + "RequestMethod": "POST", + "RequestURL": "#{protocol}://#{host}/upload", + "Body": "MultipartFormData", + "FileFormName": "file", + "URL": "{json:link}", + "DeletionURL": "{json:deleteLink}", + "ErrorMessage": "{json:error}" +}) + end +end diff --git a/src/routing.cr b/src/routing.cr index c9bd297..39c4d43 100644 --- a/src/routing.cr +++ b/src/routing.cr @@ -20,6 +20,12 @@ module Routing Handling.upload_url(env) end + if CONFIG.adminEnabled + post "/api/admin/delete" do |env| + Handling::Admin.delete_file(env) + end + end + get "/:filename" do |env| Handling.retrieve_file(env) end diff --git a/src/utils.cr b/src/utils.cr index 33e591d..256ecb1 100644 --- a/src/utils.cr +++ b/src/utils.cr @@ -29,7 +29,7 @@ module Utils def create_thumbnails_dir if !CONFIG.thumbnails if !Dir.exists?("#{CONFIG.thumbnails}") - LOGGER.info "Creating thumbnaisl folder under '#{CONFIG.thumbnails}'" + LOGGER.info "Creating thumbnails folder under '#{CONFIG.thumbnails}'" begin Dir.mkdir("#{CONFIG.thumbnails}") rescue ex @@ -145,6 +145,25 @@ module Utils end end + def delete_file(env) + fileinfo = SQL.query_all("SELECT filename, extension, thumbnail + FROM #{CONFIG.db_table_name} + WHERE delete_key = ?", + env.params.query["key"], + as: {filename: String, extension: String, thumbnail: String | Nil})[0] + + # Delete file + File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}") + if fileinfo[:thumbnail] + # Delete thumbnail + File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") + end + # Delete entry from db + SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"] + LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" + msg("File '#{fileinfo[:filename]}' deleted successfully") + end + def detect_extension(file) : String magic_bytes = { ".png" => "89504e470d0a1a0a", diff --git a/src/views/index.ecr b/src/views/index.ecr index 157696a..bd152ee 100644 --- a/src/views/index.ecr +++ b/src/views/index.ecr @@ -22,7 +22,7 @@

- Chatterino Config | ShareX Config | file-uploader-crystal (BETA <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %>) + Chatterino Config | ShareX Config | file-uploader-crystal (BETA <%= CURRENT_TAG %> - <%= CURRENT_VERSION %> @ <%= CURRENT_BRANCH %>)

Archivos alojados: <%= files_hosted %>