From 6f5e5c2007203d93d6d13438837bcf6876b5a447 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Tue, 6 Aug 2024 18:42:53 -0400 Subject: [PATCH] 0.7.0: Use standarized headers, automatic protocol detection and usage documentation --- README.md | 64 ++++++++++++++++++++++-- config/config.yml | 4 +- shard.yml | 4 +- src/config.cr | 1 - src/file-uploader.cr | 6 +++ src/handling.cr | 116 ++++++++++++++++--------------------------- src/http-errors.cr | 40 +++++++++++++++ src/utils.cr | 13 +++++ 8 files changed, 164 insertions(+), 84 deletions(-) create mode 100644 src/http-errors.cr diff --git a/README.md b/README.md index 4d16383..504b512 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,76 @@ # file-uploader Simple file uploader made on Crystal. -~~I'm making this to replace my current File uploader hosted on https://ayaya.beauty which uses https://github.com/nokonoko/uguu~~ Already replaced lol. +~~I'm making this to replace my current File uploader hosted on https://ayaya.beauty which uses https://github.com/nokonoko/uguu~~ +Already replaced lol. ## Features -- Temporary file file uploader like Uguu +- Temporary file uploads like Uguu - File deletion link (not available in frontend for now) - Chatterino and ShareX support - Unix socket support if you don't want to deal with all the TCP overhead -- Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. I will depend of your traffic. +- Automatic protocol detection (HTTPS or HTTP) +- Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. It will depend of your traffic. + +## Usage + +- Clone this repository, compile it using `shards build --release` and execute the server using `./bin/file-uploader`. +- Change the settings file `./config/config.yml` acording to what you need. + +## NGINX Server block + +Assuming you are already using NGINX and you know how to use it, you can use this example server block. + +``` +server { + # You can keep the domain prefixed with `~.` if you want + # to allow users to use any domain to upload and retrieve + # files. Like xdxd.example.com or lolol.example.com . + # This will only work if you have a wildcard domain. + server_name ~.example.com example.com; + + location / { + proxy_pass http://127.0.0.1:8080; + # This if you want to use a UNIX socket instead + #proxy_pass http://unix:/tmp/file-uploader.sock; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_pass_request_headers on; + } + + # This should be the size_limit value (from config.yml) + client_max_body_size 512M; + + listen 443 ssl; + http2 on; +} +``` +## Systemd user service example + +``` +[Unit] +Description=file-uploader-crystal +After=network.target + +[Service] +Type=simple +Restart=always +RestartSec=2 +LimitNOFILE=4096 +Environment="KEMAL_ENV=production" +ExecStart=%h/file-uploader-crystal/bin/file-uploader +WorkingDirectory=%h/file-uploader-crystal/ + +[Install] +WantedBy=default.target +``` ## TODO - ~~Add file size limit~~ ADDED -- Fix error when accessing `http://127.0.0.1:8080` with an empty DB. +- ~~Fix error when accessing `http://127.0.0.1:8080` with an empty DB.~~ Fixed somehow. - Better frontend... - ~~Disable file deletion if `delete_files_after_check_seconds` or `delete_files_after` is set to `0`~~ DONE - ~~Disable delete key if `delete_key_length` is `0`~~ DONE (But I think there is a better way to do it) diff --git a/config/config.yml b/config/config.yml index 148e9cd..b3735dc 100644 --- a/config/config.yml +++ b/config/config.yml @@ -1,6 +1,4 @@ files: "./files" -# Set to true if running behind a reverse proxy like nignx -secure: true db: "./db.sqlite3" db_table_name: "files" filename_length: 3 @@ -8,7 +6,7 @@ filename_length: 3 size_limit: 512 port: 8080 # If you define the unix socket, it will only listen on the socket and not the port. -unix_socket: "/tmp/file-uploader.sock" +#unix_socket: "/tmp/file-uploader.sock" # In days delete_files_after: 7 # In seconds diff --git a/shard.yml b/shard.yml index 15c2d0f..9e72249 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: file-uploader -version: 0.1.0 +version: 0.7.0 authors: - Fijxu @@ -16,4 +16,4 @@ dependencies: crystal: '>= 1.12.2' -license: MIT +license: AGPL-3.0-only diff --git a/src/config.cr b/src/config.cr index 2d1e35d..d98fecc 100644 --- a/src/config.cr +++ b/src/config.cr @@ -4,7 +4,6 @@ class Config include YAML::Serializable property files : String = "./files" - property secure : Bool = false property db : String = "./db.sqlite3" property db_table_name : String = "files" property filename_length : Int8 = 3 diff --git a/src/file-uploader.cr b/src/file-uploader.cr index 95751d0..60bcc69 100644 --- a/src/file-uploader.cr +++ b/src/file-uploader.cr @@ -33,6 +33,7 @@ Routing.register_all Jobs.run +Utils.delete_socket # Simple but ugly way if !CONFIG.unix_socket.nil? Kemal.run do |config| @@ -42,6 +43,11 @@ else Kemal.run end +# Set permissions to 777 so NGINX can read and write to it (BROKEN) +sleep 1.second +LOGGER.info "Setting sock permissions to 777" +File.chmod("#{CONFIG.unix_socket}", File::Permissions::All) + {% if flag?(:release) || flag?(:production) %} Kemal.config.env = "production" if !ENV.has_key?("KEMAL_ENV") {% end %} diff --git a/src/handling.cr b/src/handling.cr index 86f5180..a7f5f89 100644 --- a/src/handling.cr +++ b/src/handling.cr @@ -1,50 +1,13 @@ +require "./http-errors" + module Handling extend self - private macro error401(message) - env.response.content_type = "application/json" - env.response.status_code = 401 - error_message = {"error" => {{message}}}.to_json - return error_message - end - - private macro error403(message) - env.response.content_type = "application/json" - env.response.status_code = 403 - error_message = {"error" => {{message}}}.to_json - return error_message - end - - private macro error404(message) - env.response.content_type = "application/json" - env.response.status_code = 404 - error_message = {"error" => {{message}}}.to_json - return error_message - end - - private macro error413(message) - env.response.content_type = "application/json" - env.response.status_code = 413 - error_message = {"error" => {{message}}}.to_json - return error_message - end - - private macro error500(message) - env.response.content_type = "application/json" - env.response.status_code = 500 - error_message = {"error" => {{message}}}.to_json - return error_message - end - - private macro msg(message) - env.response.content_type = "application/json" - msg = {"message" => {{message}}}.to_json - return msg -end - def upload(env) env.response.content_type = "application/json" - # You can modify this if you want to allow files smaller than 1MiB + # You can modify this if you want to allow files smaller than 1MiB. + # This is generally a good way to check the filesize but there is a better way to do it + # which is inspecting the file directly (If I'm not wrong). if CONFIG.size_limit > 0 if env.request.headers["Content-Length"].to_i > 1048576*CONFIG.size_limit error413("File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB") @@ -57,9 +20,6 @@ end file_hash = "" ip_address = "" delete_key = nil - if CONFIG.delete_key_length > 0 - delete_key = Random.base58(CONFIG.delete_key_length) - end # TODO: Return the file that matches a checksum inside the database HTTP::FormData.parse(env.request) do |upload| next if upload.filename.nil? || upload.filename.to_s.empty? @@ -67,58 +27,65 @@ end if CONFIG.blocked_extensions.includes?(extension.split(".")[1]) error401("Extension '#{extension}' is not allowed") end - # TODO: Check if random string is already taken by some file (This will likely - # never happen but it is better to design it that way) - # filename = Random.base58(CONFIG.filename_length) filename = Utils.generate_filename - if !filename.is_a?(String) - error403("This doesn't look like a file") - else - file_path = ::File.join ["#{CONFIG.files}", filename + extension] - File.open(file_path, "w") do |file| - IO.copy(upload.body, file) - end - - original_filename = upload.filename - uploaded_at = Time.utc - file_hash = Utils.hash_file(file_path) - ip_address = env.request.remote_address.to_s.split(":").first - SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?)", - original_filename, filename, extension, uploaded_at, file_hash, ip_address, delete_key + file_path = ::File.join ["#{CONFIG.files}", filename + extension] + File.open(file_path, "w") do |file| + IO.copy(upload.body, file) end + original_filename = upload.filename + uploaded_at = Time.utc + file_hash = Utils.hash_file(file_path) + # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse + # proxy configuration. + ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first end if !filename.empty? - JSON.build do |j| + protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" + host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] + json = JSON.build do |j| j.object do - CONFIG.secure ? j.field "link", "https://#{env.request.headers["Host"]}/#{filename}" : j.field "link", "http://#{env.request.headers["Host"]}/#{filename}" - j.field "linkExt", "https://#{env.request.headers["Host"]}/#{filename}#{extension}" + j.field "link", "#{protocol}://#{host}/#{filename}" + j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}" j.field "id", filename j.field "ext", extension j.field "name", original_filename j.field "checksum", file_hash if CONFIG.delete_key_length > 0 + delete_key = Random.base58(CONFIG.delete_key_length) j.field "deleteKey", delete_key - j.field "deleteLink", "https://#{env.request.headers["Host"]}/delete?key=#{delete_key}" + j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}" end end end + begin + # Insert SQL data just before returning the upload information + SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?)", + original_filename, filename, extension, uploaded_at, file_hash, ip_address, delete_key + rescue ex + LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" + error500("An error ocurred when trying to insert the data into the DB") + end + return json else - error403("No file") + LOGGER.debug "No file provided by the user" + error403("No file provided") end end def retrieve_file(env) begin - LOGGER.debug "#{env.request.headers["X-Real-IP"]} /#{env.params.url["filename"]}" + LOGGER.debug "#{env.request.headers["X-Forwarded-For"]} /#{env.params.url["filename"]}" rescue - LOGGER.debug "NO X-Real-IP @ /#{env.params.url["filename"]}" + LOGGER.debug "NO X-Forwarded-For @ /#{env.params.url["filename"]}" end begin filename = SQL.query_one "SELECT filename FROM #{CONFIG.db_table_name} WHERE filename = ?", env.params.url["filename"].to_s.split(".").first, as: String + original_filename = SQL.query_one "SELECT original_filename FROM #{CONFIG.db_table_name} WHERE filename = ?", env.params.url["filename"].to_s.split(".").first, as: String extension = SQL.query_one "SELECT extension FROM #{CONFIG.db_table_name} WHERE filename = ?", filename, as: String + headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{original_filename}"}) send_file env, "#{CONFIG.files}/#{filename}#{extension}" - rescue - LOGGER.debug "File #{filename} does not exists" + rescue ex + LOGGER.debug "File #{filename} does not exist: #{ex.message}" error403("File #{filename} does not exist") end end @@ -137,7 +104,7 @@ end end end rescue ex - LOGGER.error "#{ex.message}" + LOGGER.error "Unknown error: #{ex.message}" error500("Unknown error") end json_data @@ -153,11 +120,12 @@ end LOGGER.debug "File '#{file_to_delete}' was deleted using key '#{env.params.query["key"]}'}" msg("File '#{file_to_delete}' deleted successfully") rescue ex - error500("Unknown error: #{ex.message}") + LOGGER.error("Unknown error: #{ex.message}") + error500("Unknown error") end else LOGGER.debug "Key '#{env.params.query["key"]}' does not exist" - error401("Huh? This delete key doesn't exist") + error401("Delete key '#{env.params.query["key"]}' does not exist. No files were deleted") end end end diff --git a/src/http-errors.cr b/src/http-errors.cr new file mode 100644 index 0000000..023d3a5 --- /dev/null +++ b/src/http-errors.cr @@ -0,0 +1,40 @@ +macro error401(message) + env.response.content_type = "application/json" + env.response.status_code = 401 + error_message = {"error" => {{message}}}.to_json + return error_message + end + +macro error403(message) + env.response.content_type = "application/json" + env.response.status_code = 403 + error_message = {"error" => {{message}}}.to_json + return error_message + end + +macro error404(message) + env.response.content_type = "application/json" + env.response.status_code = 404 + error_message = {"error" => {{message}}}.to_json + return error_message + end + +macro error413(message) + env.response.content_type = "application/json" + env.response.status_code = 413 + error_message = {"error" => {{message}}}.to_json + return error_message + end + +macro error500(message) + env.response.content_type = "application/json" + env.response.status_code = 500 + error_message = {"error" => {{message}}}.to_json + return error_message + end + +macro msg(message) + env.response.content_type = "application/json" + msg = {"message" => {{message}}}.to_json + return msg +end diff --git a/src/utils.cr b/src/utils.cr index 370e013..d3e0504 100644 --- a/src/utils.cr +++ b/src/utils.cr @@ -69,4 +69,17 @@ module Utils end end end + + # Delete socket if the server has not been previously cleaned by the server (Due to unclean exits, crashes, etc.) + def delete_socket + if File.exists?("#{CONFIG.unix_socket}") + LOGGER.info "Deleting old unix socket" + begin + File.delete("#{CONFIG.unix_socket}") + rescue ex + LOGGER.fatal "#{ex.message}" + exit(1) + end + end + end end