diff --git a/.gitignore b/.gitignore index 2bce430..94285f5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ data torexitnodes.txt files thumbnails +db.sqlite3 \ No newline at end of file diff --git a/README.md b/README.md index 76b35b3..b9e1ccc 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # file-uploader +> [!WARNING] +> Project being rewritten, some features like the admin API and some upload endpoints are unavailable on 0.9.5 + Simple file uploader made on Crystal. ~~I'm making this to replace my current File uploader hosted on https://ayaya.beauty which uses https://github.com/nokonoko/uguu~~ Already replaced lol. @@ -9,7 +12,7 @@ Already replaced lol. - Temporary file uploads like Uguu - File deletion link (not available in frontend for now) - Chatterino and ShareX support -- Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, can be disabled.) +- Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, disabled by default) - Rate Limiting - [Small Admin API](./src/handling/admin.cr) that allows you to delete files, reset rate limits and more (Needs to be enabled in the configuration) - Unix socket support if you don't want to deal with all the TCP overhead @@ -37,7 +40,7 @@ server { proxy_pass http://127.0.0.1:8080; # This if you want to use a UNIX socket instead #proxy_pass http://unix:/tmp/file-uploader.sock; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Host $host; proxy_pass_request_headers on; @@ -69,20 +72,3 @@ WorkingDirectory=%h/file-uploader-crystal/ [Install] WantedBy=default.target ``` - -## TODO - -- ~~Add file size limit~~ ADDED -- ~~Fix error when accessing `http://127.0.0.1:8080` with an empty DB.~~ Fixed somehow. -- Better frontend... -- ~~Disable file deletion if `deleteFilesCheck` or `deleteFilesAfter` is set to `0`~~ DONE -- ~~Disable delete key if `deleteKeyLength` is `0`~~ DONE (But I think there is a better way to do it) -- ~~Exit if `fileameLength` is `0`~~ DONE -- ~~Disable file limit if `size_limit` is `0`~~ DONE -- ~~Prevent files from being overwritten in the event of a name collision~~ DONE -- Dockerfile and Docker image (Crystal doesn't has dependency hell like other languages so is not really necessary to do, but useful for people that want instant deploy) -- Custom file expiration using headers (Like rustypaste) -- Small CLI to upload files (like `rpaste` from rustypaste) -- Add more endpoints to Admin API - -- diff --git a/config/config.example.yml b/config/config.example.yml index 18f9602..d3e921d 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -1,11 +1,11 @@ colorize_logs: true files: "./files" thumbnails: "./thumbnails" -generateThumbnails: true +generate_thumbnails: true db: "./db/db.sqlite3" adminEnabled: true adminApiKey: "asd" -fileameLength: 3 +filename_length: 3 # In MiB size_limit: 512 port: 8080 @@ -42,5 +42,5 @@ opengraphUseragents: - "Mastodon/" # You can leave it empty, or add your own domains. -alternativeDomains: +alternative_domains: - "example.com" diff --git a/public/bliss-small.avif b/public/bliss-small.avif deleted file mode 100644 index 77c176c..0000000 Binary files a/public/bliss-small.avif and /dev/null differ diff --git a/public/favicon.gif b/public/favicon.gif deleted file mode 100644 index 1cae26f..0000000 Binary files a/public/favicon.gif and /dev/null differ diff --git a/public/favicon.ico b/public/favicon.ico index 6ec8a8c..b4d8eac 100644 Binary files a/public/favicon.ico and b/public/favicon.ico differ diff --git a/public/favicon.png b/public/favicon.png new file mode 100644 index 0000000..22c89ba Binary files /dev/null and b/public/favicon.png differ diff --git a/public/script.js b/public/script.js index 255e99f..111e566 100644 --- a/public/script.js +++ b/public/script.js @@ -89,18 +89,18 @@ document.addEventListener("DOMContentLoaded", () => { uploadContainer.className = "upload-status"; // Use the existing CSS class for styling uploadContainer.appendChild(uploadText); uploadContainer.appendChild(statusLink); - buttons.appendChild(copyButton) - buttons.appendChild(deleteButton) - uploadContainer.appendChild(buttons) + buttons.appendChild(copyButton) + buttons.appendChild(deleteButton) + uploadContainer.appendChild(buttons) uploadStatus.appendChild(uploadContainer); // Update upload text uploadText.innerHTML = "0%"; uploadText.className = "percent"; statusLink.className = "status"; - copyButton.className = "copy-button"; // Add class for styling + copyButton.className = "button copy-button"; // Add class for styling copyButton.innerHTML = "Copiar"; // Set button text - deleteButton.className = "delete-button"; + deleteButton.className = "button delete-button"; deleteButton.innerHTML = "Borrar"; copyButton.style.display = "none"; deleteButton.style.display = "none"; diff --git a/public/styles.css b/public/styles.css index 46d7799..3da918d 100644 --- a/public/styles.css +++ b/public/styles.css @@ -18,10 +18,7 @@ html { font-family: "FG"; - background-image: linear-gradient(to bottom, - rgba(11, 11, 11, 0.92), - rgba(11, 11, 11, 0.92)), - url(./bliss-small.avif); + background: #111111; background-attachment: fixed; background-repeat: no-repeat; background-size: cover; @@ -29,8 +26,6 @@ html { body { - /* font-family: Arial, sans-serif; */ - /* background-color: #111; */ margin: 0; padding: 20px; } @@ -53,11 +48,11 @@ h1 { a { text-decoration: none; + color: #ffb6c1 } .bottom { font-size: 0.9em; - /* margin-top: 1ch;*/ flex: 1; text-align: center; } @@ -73,31 +68,25 @@ a { .container { max-width: 800px; margin: auto; - /* background: white; */ - /*! padding: 20px; */ border-radius: 0px; - /*! box-shadow: 0 0 10px rgba(0, 0, 0, 0.1); */ } +.container>img { + max-width: 100%; + max-height: 500px +} #drop-area { - /*! border: 2px solid #00ff00; */ - /*! border-radius: 6px; */ - /*! padding-left: 10px; */ - /*! padding-right: 10px; */ text-align: center; position: relative; width: fit-content; margin: 0 auto; - /* Center the element */ display: block; - /* Ensure it behaves as a block-level element */ background: rgba(202, 230, 190, .75); border: 1px solid #b7d1a0; border-radius: 4px; color: #468847; cursor: pointer; - /*! display: inline-block; */ font-size: 24px; padding: 28px 48px; text-shadow: 0 1px hsla(0, 0%, 100%, .5); @@ -107,11 +96,9 @@ a { .button { display: inline-block; padding: 10px 20px; - /* background: #; */ color: white; border-radius: 5px; cursor: pointer; - /* margin-top: 10px; */ } .upload-status { @@ -128,101 +115,59 @@ nav>ul { #upload-status { margin: 20px; - /* Adjust as needed */ } .upload-status { display: flex; align-items: center; justify-content: space-between; - border: 2px solid #999; - /* Optional styling for the status box */ + border: 1px solid #ffffff; padding: 5px; - /* Optional padding */ - /*! border-radius: 6px; */ - /* Optional rounded corners */ - /*! background-color: #f9f9f9; */ - /* Optional background color */ } .link-container { display: flex; align-items: center; margin-left: auto; - /* Pushes the link and button to the right */ } .link { color: #ffb6c1; text-decoration: none; - /* Remove underline from link */ margin-right: 5px; - /* Space between link and button */ } .link:hover { text-decoration: underline; - /* Optional: underline on hover */ +} + +.button { + display: inline; + color: rgb(255, 255, 255); + border: none; + border-radius: 3px; + padding: 5px 10px; + cursor: pointer; + font-weight: bold; } .copy-button { - display: inline; - background-color: #7a6fff; - /* Button background color */ - color: white; - /* Button text color */ - border: none; - /* Remove border */ - border-radius: 3px; - /* Rounded corners for the button */ - padding: 5px 10px; - /* Button padding */ - cursor: pointer; - /* Pointer cursor on hover */ - font-weight: bold; + background-color: #1e8a1a; } .delete-button { - display: inline; - background-color: #ff6f6f; - /* Button background color */ - color: white; - /* Button text color */ - border: none; - /* Remove border */ - border-radius: 3px; - /* Rounded corners for the button */ - padding: 5px 10px; - /* Button padding */ - cursor: pointer; - /* Pointer cursor on hover */ - margin-left: 6px; - font-weight: bold; + background-color: #b83434; + margin-left: 6px } - .copy-button:hover { - background-color: #6057ce; - /* Darker shade on hover */ + background-color: #156412; } .delete-button:hover { - background-color: #ce5757; - /* Darker shade on hover */ + background-color: #912a2a; } .status { color: rgb(255, 132, 0); -} - -a:link { - color: #ffb6c1 -} - -a:visited { - color: #ffb6c1 -} - -a:hover { - color: #ffb6c1 } \ No newline at end of file diff --git a/src/config.cr b/src/config.cr index 1fbd8ce..eccdb4e 100644 --- a/src/config.cr +++ b/src/config.cr @@ -2,36 +2,43 @@ require "yaml" class Config include YAML::Serializable - # Colorize logs property colorize_logs : Bool = true + # Log level + property log_level : LogLevel = LogLevel::Info + + # Port on which the uploader will bind + property port : Int32 = 8080 + # IP address on which the uploader will bind + property host : String = "127.0.0.1" + # Where the uploaded files will be located property files : String = "./files" # Where the thumbnails will be located when they are successfully generated property thumbnails : String = "./thumbnails" # Generate thumbnails for OpenGraph compatible platforms like Chatterino # Whatsapp, Facebook, Discord, etc. - property generateThumbnails : Bool = false + property generate_thumbnails : Bool = false # Where the SQLITE3 database will be located property db : String = "./db.sqlite3" + # Enable or disable the admin API property adminEnabled : Bool = false # The API key for admin routes. It's passed as a "X-Api-Key" header to the # request property adminApiKey : String? = "" + # Not implemented - property incrementalFileameLength : Bool = true + property incrementalfilename_length : Bool = true # Filename length - property fileameLength : Int32 = 3 + property filename_length : Int32 = 3 # In MiB property size_limit : Int16 = 512 - # Port on which the uploader will bind - property port : Int32 = 8080 - # IP address on which the uploader will bind - property host : String = "127.0.0.1" + # A file path where do you want to place a unix socket (THIS WILL DISABLE ACCESS # BY IP ADDRESS) property unix_socket : String? + # True if you want this program to block IP addresses coming from the Tor # network property blockTorAddresses : Bool? = false @@ -39,40 +46,42 @@ class Config property torExitNodesCheck : Int32 = 3600 # Only https://check.torproject.org/exit-addresses is supported property torExitNodesUrl : String = "https://check.torproject.org/exit-addresses" - # Where the file of the exit nodes will be located, can be placed anywhere - property torExitNodesFile : String = "./torexitnodes.txt" # Message that will be displayed to the Tor user. # It will be shown on the Frontend and shown in the error 401 when a user # tries to upload a file using curl or any other tool property torMessage : String? = "Tor is blocked!" + # How many files an IP address can upload to the server property filesPerIP : Int32 = 32 # How often is the file limit per IP reset? (in seconds) property rateLimitPeriod : Int32 = 600 # TODO: UNUSED CONSTANT property rateLimitMessage : String = "" + # Delete the files after how many days? - property deleteFilesAfter : Int32 = 7 + property deleteFilesAfter : Int32 = 14 # How often should the check of old files be performed? (in seconds) property deleteFilesCheck : Int32 = 1800 # The lenght of the delete key - property deleteKeyLength : Int32 = 4 + property deleteKeyLength : Int32 = 6 + property siteInfo : String = "xd" # TODO: UNUSED CONSTANT property siteWarning : String? = "" - # Log level - property log_level : LogLevel = LogLevel::Info + # Blocked extensions that are not allowed to be uploaded to the server property blockedExtensions : Array(String) = [] of String + # A list of OpenGraph user agents. If the request contains one of those User # agents when trying to retrieve a file from the server; the server will # reply with an HTML with OpenGraph tags, pointing to the media thumbnail # (if it was generated successfully) and the name of the file as title property opengraphUseragents : Array(String) = [] of String + # Since this program detects the Host header of the client it can be used # with multiple domains. You can display the domains in the frontend # and in `/api/stats` - property alternativeDomains : Array(String) = [] of String + property alternative_domains : Array(String) = [] of String def self.load config_file = "config/config.yml" @@ -83,16 +92,16 @@ class Config end def self.check_config(config : Config) - if config.fileameLength <= 0 - puts "Config: fileameLength cannot be #{config.fileameLength}" + if config.filename_length <= 0 + puts "Config: filename_length cannot be less or equal to 0" exit(1) end - if config.files.ends_with?('/') + if config.files.ends_with?('/') config.files = config.files.chomp('/') end - if config.thumbnails.ends_with?('/') - config.thumbnails = config.thumbnails.chomp('/') - end + if config.thumbnails.ends_with?('/') + config.thumbnails = config.thumbnails.chomp('/') + end end end diff --git a/src/database/files.cr b/src/database/files.cr new file mode 100644 index 0000000..3b15bd3 --- /dev/null +++ b/src/database/files.cr @@ -0,0 +1,83 @@ +module Database::Files + extend self + + # ------------------- + # Insert / Delete + # ------------------- + + def insert(file : UFile) : Nil + request = <<-SQL + INSERT INTO files + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT DO NOTHING + SQL + + SQL.exec(request, *file.to_tuple) + end + + def delete(filename : String) : Nil + request = <<-SQL + DELETE + FROM files + WHERE filename = ? + SQL + + SQL.exec(request, filename) + end + + def delete_with_key(key : String) : Nil + request = <<-SQL + DELETE FROM files + WHERE delete_key = ? + SQL + + SQL.exec(request, key) + end + + # ------------------- + # Select + # ------------------- + + def select(filename : String) : UFile? + request = <<-SQL + SELECT * + FROM files + WHERE filename = ? + SQL + + SQL.query_one?(request, filename, as: UFile) + end + + def select_with_key(delete_key : String) : UFile? + request = <<-SQL + SELECT * + FROM files + WHERE delete_key = ? + SQL + + SQL.query_one?(request, delete_key, as: UFile) + end + + # ------------------- + # Misc + # ------------------- + + def old_files : Array(UFile) + request = <<-SQL + SELECT filename, extension, thumbnail + FROM files + WHERE uploaded_at < strftime('%s', 'now') - #{CONFIG.deleteFilesAfter * 3600} + SQL + + SQL.query_all(request, as: UFile) + end + + def file_count : Int32 + request = <<-SQL + SELECT COUNT (filename) + FROM files + SQL + + SQL.query_one(request, as: Int32) + end +end diff --git a/src/database/ip.cr b/src/database/ip.cr new file mode 100644 index 0000000..c818185 --- /dev/null +++ b/src/database/ip.cr @@ -0,0 +1,16 @@ +module Database::IP + # ------------------- + # Insert / Delete + # ------------------- + + def insert(ip : IP) : Nil + request = <<-SQL + INSERT OR IGNORE + INTO ips (ip, date) + VALUES ($1, $2) + ON CONFLICT DO NOTHING + SQL + + SQL.exec(request, *ip.to_tuple) + end +end diff --git a/src/file-uploader-crystal.cr b/src/file-uploader-crystal.cr index 850c4a2..b333235 100644 --- a/src/file-uploader-crystal.cr +++ b/src/file-uploader-crystal.cr @@ -7,11 +7,12 @@ require "digest" require "./logger" require "./routing" -require "./utils" -require "./handling/**" require "./config" require "./jobs" -require "./lib/**" +require "./utils/*" +require "./lib/*" +require "./types/*" +require "./database/*" CONFIG = Config.load Kemal.config.port = CONFIG.port @@ -21,7 +22,7 @@ Kemal.config.app_name = "file-uploader-crystal" # https://github.com/iv-org/invidious/blob/90e94d4e6cc126a8b7a091d12d7a5556bfe369d5/src/invidious.cr#L136C1-L136C61 LOGGER = LogHandler.new(STDOUT, CONFIG.log_level, CONFIG.colorize_logs) # Give me a 128 bit CPU -# MAX_FILES = 58**CONFIG.fileameLength +# MAX_FILES = 58**CONFIG.filename_length SQL = DB.open("sqlite3://#{CONFIG.db}") # https://github.com/iv-org/invidious/blob/90e94d4e6cc126a8b7a091d12d7a5556bfe369d5/src/invidious.cr#L78 diff --git a/src/handling/handling.cr b/src/handling/handling.cr deleted file mode 100644 index be6f554..0000000 --- a/src/handling/handling.cr +++ /dev/null @@ -1,401 +0,0 @@ -require "../http-errors" -require "http/client" -require "benchmark" - -# require "../filters" - -module Handling - extend self - - def upload(env) - env.response.content_type = "application/json" - ip_address = Utils.ip_address(env) - protocol = Utils.protocol(env) - host = Utils.host(env) - # filter = env.params.query["filter"]? - # You can modify this if you want to allow files smaller than 1MiB. - # This is generally a good way to check the filesize but there is a better way to do it - # which is inspecting the file directly (If I'm not wrong). - if CONFIG.size_limit > 0 - if env.request.headers["Content-Length"].to_i > 1048576*CONFIG.size_limit - return http_error 413, "File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB" - end - end - filename = "" - extension = "" - original_filename = "" - uploaded_at = "" - checksum = "" - if CONFIG.deleteKeyLength > 0 - delete_key = Random.base58(CONFIG.deleteKeyLength) - end - # TODO: Return the file that matches a checksum inside the database - HTTP::FormData.parse(env.request) do |upload| - if upload.filename.nil? || upload.filename.to_s.empty? - LOGGER.debug "No file provided by the user" - return http_error 403, "No file provided" - end - # TODO: upload.body is emptied when is copied or read - # Utils.check_duplicate(upload.dup) - extension = File.extname("#{upload.filename}") - if CONFIG.blockedExtensions.includes?(extension.split(".")[1]) - return http_error 401, "Extension '#{extension}' is not allowed" - end - filename = Utils.generate_filename - file_path = "#{CONFIG.files}/#{filename}#{extension}" - File.open(file_path, "w") do |output| - IO.copy(upload.body, output) - end - original_filename = upload.filename - uploaded_at = Time.utc - checksum = Utils.hash_file(file_path) - # TODO: Apply filters - # if filter - # Filters.apply_filter(file_path, filter) - # end - end - # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse - # proxy configuration. - begin - spawn { Utils.generate_thumbnail(filename, extension) } - rescue ex - LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" - end - begin - # Insert SQL data just before returning the upload information - SQL.exec "INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil - SQL.exec "INSERT OR IGNORE INTO ips (ip, date) VALUES (?, ?)", ip_address, Time.utc.to_unix - # SQL.exec "INSERT OR IGNORE INTO ips (ip) VALUES ('#{ip_address}')" - SQL.exec "UPDATE ips SET count = count + 1 WHERE ip = ('#{ip_address}')" - rescue ex - LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" - return http_error 500, "An error ocurred when trying to insert the data into the DB" - end - json = JSON.build do |j| - j.object do - j.field "link", "#{protocol}://#{host}/#{filename}" - j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}" - j.field "id", filename - j.field "ext", extension - j.field "name", original_filename - j.field "checksum", checksum - if CONFIG.deleteKeyLength > 0 - j.field "deleteKey", delete_key - j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}" - end - end - end - json - end - - # The most unoptimized and unstable feature lol - def upload_url_bulk(env) - env.response.content_type = "application/json" - ip_address = Utils.ip_address(env) - protocol = Utils.protocol(env) - host = Utils.host(env) - begin - files = env.params.json["files"].as((Array(JSON::Any))) - rescue ex : JSON::ParseException - LOGGER.error "Body malformed: #{ex.message}" - return http_error 400, "Body malformed: #{ex.message}" - rescue ex - LOGGER.error "Unknown error: #{ex.message}" - return http_error 500, "Unknown error" - end - successfull_files = [] of NamedTuple(filename: String, extension: String, original_filename: String, checksum: String, delete_key: String | Nil) - failed_files = [] of String - # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse - # proxy configuration. - files.each do |url| - url = url.to_s - filename = Utils.generate_filename - original_filename = "" - extension = "" - checksum = "" - uploaded_at = Time.utc - extension = File.extname(URI.parse(url).path) - if CONFIG.deleteKeyLength > 0 - delete_key = Random.base58(CONFIG.deleteKeyLength) - end - file_path = "#{CONFIG.files}/#{filename}#{extension}" - File.open(file_path, "w") do |output| - begin - HTTP::Client.get(url) do |res| - IO.copy(res.body_io, output) - end - rescue ex - LOGGER.debug "Failed to download file '#{url}': #{ex.message}" - return http_error 403, "Failed to download file '#{url}'" - failed_files << url - end - end - # successfull_files << url - # end - if extension.empty? - extension = Utils.detect_extension(file_path) - File.rename(file_path, file_path + extension) - file_path = "#{CONFIG.files}/#{filename}#{extension}" - end - # The second one is faster and it uses less memory - # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last - original_filename = url.split("/").last - checksum = Utils.hash_file(file_path) - begin - spawn { Utils.generate_thumbnail(filename, extension) } - rescue ex - LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" - end - begin - # Insert SQL data just before returning the upload information - SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil) - successfull_files << {filename: filename, - original_filename: original_filename, - extension: extension, - delete_key: delete_key, - checksum: checksum} - rescue ex - LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" - return http_error 500, "An error ocurred when trying to insert the data into the DB" - end - end - json = JSON.build do |j| - j.array do - successfull_files.each do |fileinfo| - j.object do - j.field "link", "#{protocol}://#{host}/#{fileinfo[:filename]}" - j.field "linkExt", "#{protocol}://#{host}/#{fileinfo[:filename]}#{fileinfo[:extension]}" - j.field "id", fileinfo[:filename] - j.field "ext", fileinfo[:extension] - j.field "name", fileinfo[:original_filename] - j.field "checksum", fileinfo[:checksum] - if CONFIG.deleteKeyLength > 0 - delete_key = Random.base58(CONFIG.deleteKeyLength) - j.field "deleteKey", fileinfo[:delete_key] - j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{fileinfo[:delete_key]}" - end - end - end - end - end - json - end - - def upload_url(env) - env.response.content_type = "application/json" - ip_address = Utils.ip_address(env) - protocol = Utils.protocol(env) - host = Utils.host(env) - url = env.params.query["url"] - successfull_files = [] of NamedTuple(filename: String, extension: String, original_filename: String, checksum: String, delete_key: String | Nil) - failed_files = [] of String - # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse - # proxy configuration. - filename = Utils.generate_filename - original_filename = "" - extension = "" - checksum = "" - uploaded_at = Time.utc - extension = File.extname(URI.parse(url).path) - if CONFIG.deleteKeyLength > 0 - delete_key = Random.base58(CONFIG.deleteKeyLength) - end - file_path = "#{CONFIG.files}/#{filename}#{extension}" - File.open(file_path, "w") do |output| - begin - # TODO: Connect timeout to prevent possible Denial of Service to the external website spamming requests - # https://crystal-lang.org/api/1.13.2/HTTP/Client.html#connect_timeout - HTTP::Client.get(url) do |res| - IO.copy(res.body_io, output) - end - rescue ex - LOGGER.debug "Failed to download file '#{url}': #{ex.message}" - return http_error 403, "Failed to download file '#{url}': #{ex.message}" - failed_files << url - end - end - if extension.empty? - extension = Utils.detect_extension(file_path) - File.rename(file_path, file_path + extension) - file_path = "#{CONFIG.files}/#{filename}#{extension}" - end - # The second one is faster and it uses less memory - # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last - original_filename = url.split("/").last - checksum = Utils.hash_file(file_path) - begin - spawn { Utils.generate_thumbnail(filename, extension) } - rescue ex - LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" - end - begin - # Insert SQL data just before returning the upload information - SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil) - successfull_files << {filename: filename, - original_filename: original_filename, - extension: extension, - delete_key: delete_key, - checksum: checksum} - rescue ex - LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" - return http_error 500, "An error ocurred when trying to insert the data into the DB" - end - json = JSON.build do |j| - j.array do - successfull_files.each do |fileinfo| - j.object do - j.field "link", "#{protocol}://#{host}/#{fileinfo[:filename]}" - j.field "linkExt", "#{protocol}://#{host}/#{fileinfo[:filename]}#{fileinfo[:extension]}" - j.field "id", fileinfo[:filename] - j.field "ext", fileinfo[:extension] - j.field "name", fileinfo[:original_filename] - j.field "checksum", fileinfo[:checksum] - if CONFIG.deleteKeyLength > 0 - delete_key = Random.base58(CONFIG.deleteKeyLength) - j.field "deleteKey", fileinfo[:delete_key] - j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{fileinfo[:delete_key]}" - end - end - end - end - end - json - end - - def retrieve_file(env) - protocol = Utils.protocol(env) - host = Utils.host(env) - begin - fileinfo = SQL.query_one?("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail - FROM files - WHERE filename = ?", - env.params.url["filename"].split(".").first, - as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil}) - if fileinfo.nil? - # TODO: Switch this to 404, if I use 404, it will use the kemal error page (ANOYING!) - return http_error 418, "File '#{env.params.url["filename"]}' does not exist" - end - rescue ex - LOGGER.debug "Error when retrieving file '#{env.params.url["filename"]}': #{ex.message}" - return http_error 500, "Error when retrieving file '#{env.params.url["filename"]}'" - end - env.response.headers["Content-Disposition"] = "inline; filename*=UTF-8''#{fileinfo[:ofilename]}" - # env.response.headers["Last-Modified"] = "#{fileinfo[:up_at]}" - env.response.headers["ETag"] = "#{fileinfo[:checksum]}" - - CONFIG.opengraphUseragents.each do |useragent| - if env.request.headers.try &.["User-Agent"].includes?(useragent) - env.response.content_type = "text/html" - return %( - - - - - - - #{if fileinfo[:thumbnail] - %() - end} - - -) - end - end - send_file env, "#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:ext]}" - end - - def retrieve_thumbnail(env) - begin - send_file env, "#{CONFIG.thumbnails}/#{env.params.url["thumbnail"]}" - rescue ex - LOGGER.debug "Thumbnail '#{env.params.url["thumbnail"]}' does not exist: #{ex.message}" - return http_error 403, "Thumbnail '#{env.params.url["thumbnail"]}' does not exist" - end - end - - def stats(env) - env.response.content_type = "application/json" - begin - json_data = JSON.build do |json| - json.object do - json.field "stats" do - json.object do - json.field "filesHosted", SQL.query_one? "SELECT COUNT (filename) FROM files", as: Int32 - json.field "maxUploadSize", CONFIG.size_limit - json.field "thumbnailGeneration", CONFIG.generateThumbnails - json.field "filenameLength", CONFIG.fileameLength - json.field "alternativeDomains", CONFIG.alternativeDomains - end - end - end - end - rescue ex - LOGGER.error "Unknown error: #{ex.message}" - return http_error 500, "Unknown error" - end - json_data - end - - def delete_file(env) - if SQL.query_one "SELECT EXISTS(SELECT 1 FROM files WHERE delete_key = ?)", env.params.query["key"], as: Bool - begin - fileinfo = SQL.query_all("SELECT filename, extension, thumbnail - FROM files - WHERE delete_key = ?", - env.params.query["key"], - as: {filename: String, extension: String, thumbnail: String | Nil})[0] - - # Delete file - File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}") - if fileinfo[:thumbnail] - # Delete thumbnail - File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") - end - # Delete entry from db - SQL.exec "DELETE FROM files WHERE delete_key = ?", env.params.query["key"] - LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" - return msg("File '#{fileinfo[:filename]}' deleted successfully") - rescue ex - LOGGER.error("Unknown error: #{ex.message}") - return http_error 500, "Unknown error" - end - else - LOGGER.debug "Key '#{env.params.query["key"]}' does not exist" - return http_error 401, "Delete key '#{env.params.query["key"]}' does not exist. No files were deleted" - end - end - - def sharex_config(env) - host = Utils.host(env) - protocol = Utils.protocol(env) - env.response.content_type = "application/json" - # So it's able to download the file instead of displaying it - env.response.headers["Content-Disposition"] = "attachment; filename=\"#{host}.sxcu\"" - return %({ - "Version": "14.0.1", - "DestinationType": "ImageUploader, FileUploader", - "RequestMethod": "POST", - "RequestURL": "#{protocol}://#{host}/upload", - "Body": "MultipartFormData", - "FileFormName": "file", - "URL": "{json:link}", - "DeletionURL": "{json:deleteLink}", - "ErrorMessage": "{json:error}" -}) - end - - def chatterino_config(env) - host = Utils.host(env) - protocol = Utils.protocol(env) - env.response.content_type = "application/json" - return %({ - "requestUrl": "#{protocol}://#{host}/upload", - "formField": "data", - "imageLink": "{link}", - "deleteLink": "{deleteLink}" - }) - end -end diff --git a/src/http-errors.cr b/src/http-errors.cr deleted file mode 100644 index 27371d8..0000000 --- a/src/http-errors.cr +++ /dev/null @@ -1,12 +0,0 @@ -macro http_error(status_code, message) - env.response.content_type = "application/json" - env.response.status_code = {{status_code}} - error_message = {"error" => {{message}}}.to_json - error_message -end - -macro msg(message) - env.response.content_type = "application/json" - msg = {"message" => {{message}}}.to_json - msg -end diff --git a/src/jobs.cr b/src/jobs.cr index ab47716..c77c5cb 100644 --- a/src/jobs.cr +++ b/src/jobs.cr @@ -20,9 +20,7 @@ module Jobs LOGGER.info("Blocking Tor exit nodes") spawn do loop do - Utils.retrieve_tor_exit_nodes - # Updates the @@exit_nodes array instantly - Routing.reload_exit_nodes + Utils::Tor.refresh_exit_nodes sleep CONFIG.torExitNodesCheck.seconds end end diff --git a/src/logger.cr b/src/logger.cr index b613acb..39378b1 100644 --- a/src/logger.cr +++ b/src/logger.cr @@ -25,15 +25,6 @@ class LogHandler < Kemal::BaseLogHandler # Default: full path with parameters requested_url = context.request.resource - # Try not to log search queries passed as GET parameters during normal use - # (They will still be logged if log level is 'Debug' or 'Trace') - if @level > LogLevel::Debug && ( - requested_url.downcase.includes?("search") || requested_url.downcase.includes?("q=") - ) - # Log only the path - requested_url = context.request.path - end - info("#{context.response.status_code} #{context.request.method} #{requested_url} #{elapsed_text}") context diff --git a/src/macros.cr b/src/macros.cr new file mode 100644 index 0000000..972473f --- /dev/null +++ b/src/macros.cr @@ -0,0 +1,26 @@ +macro ee(status_code, message) + env.response.content_type = "application/json" + env.response.status_code = {{status_code}} + msg = {"error" => {{message}}}.to_json + return msg +end + +macro msg(message) + env.response.content_type = "application/json" + msg = {"message" => {{message}}}.to_json + return msg +end + +module Headers + macro host + env.request.headers["X-Forwarded-Host"]? + end + + macro scheme + env.request.headers["X-Forwarded-Proto"]? + end + + macro ip_addr + env.request.headers["X-Real-IP"]? + end +end diff --git a/src/handling/admin.cr b/src/routes/admin.cr similarity index 97% rename from src/handling/admin.cr rename to src/routes/admin.cr index b765386..fa66d2b 100644 --- a/src/handling/admin.cr +++ b/src/routes/admin.cr @@ -1,6 +1,4 @@ -require "../http-errors" - -module Handling::Admin +module Routes::Admin extend self # private macro json_fill(named_tuple, field_name) @@ -37,7 +35,7 @@ module Handling::Admin failed_files << file rescue ex LOGGER.error "Unknown error: #{ex.message}" - http_error 500,"Unknown error: #{ex.message}" + http_error 500, "Unknown error: #{ex.message}" end end json = JSON.build do |j| @@ -69,7 +67,7 @@ module Handling::Admin failed << item rescue ex LOGGER.error "Unknown error: #{ex.message}" - http_error 500, "Unknown error: #{ex.message}" + Macros.ee 500, "Unknown error: #{ex.message}" end end json = JSON.build do |j| @@ -107,7 +105,7 @@ module Handling::Admin failed << item rescue ex LOGGER.error "Unknown error: #{ex.message}" - http_error 500,"Unknown error: #{ex.message}" + Macros.ee 500, "Unknown error: #{ex.message}" end end json = JSON.build do |j| diff --git a/src/routes/delete.cr b/src/routes/delete.cr new file mode 100644 index 0000000..2467174 --- /dev/null +++ b/src/routes/delete.cr @@ -0,0 +1,39 @@ +module Routes::Deletion + extend self + + def delete_file(env) + key = env.params.query["key"]? + + if !key || key.empty? + ee 400, "No delete key suplied" + end + + file = Database::Files.select_with_key(key) + + if file + full_filename = file.filename + file.extension + thumbnail = file.thumbnail + + begin + # Delete file + File.delete("#{CONFIG.files}/#{full_filename}") + + if file.thumbnail + File.delete("#{CONFIG.thumbnails}/#{thumbnail}") + end + + # Delete entry from db + Database::Files.delete_with_key(key) + + LOGGER.debug "File '#{full_filename}' was deleted using key '#{key}'}" + msg("File '#{full_filename}' deleted successfully") + rescue ex + LOGGER.error("Unknown error: #{ex.message}") + ee 500, "Unknown error" + end + else + LOGGER.debug "Key '#{env.params.query["key"]}' does not exist" + ee 401, "Delete key '#{env.params.query["key"]}' does not exist. No files were deleted" + end + end +end diff --git a/src/routes/misc.cr b/src/routes/misc.cr new file mode 100644 index 0000000..6af5306 --- /dev/null +++ b/src/routes/misc.cr @@ -0,0 +1,66 @@ +require "http/client" + +module Routing::Misc + extend self + + struct Stats + include JSON::Serializable + + @[JSON::Field(key: "filesHosted")] + property files_hosted : Int32 + @[JSON::Field(key: "maxUploadSize")] + property max_upload_size : String + @[JSON::Field(key: "thumbnailGeneration")] + property thumbnail_generation : Bool + @[JSON::Field(key: "filenameLength")] + property filename_length : Int32 + @[JSON::Field(key: "alternativeDomains")] + property alternative_domains : Array(String) + + def initialize + @files_hosted = SQL.query_one("SELECT COUNT (filename) FROM files", as: Int32) + @max_upload_size = CONFIG.size_limit.to_s + @thumbnail_generation = CONFIG.generate_thumbnails + @filename_length = CONFIG.filename_length + @alternative_domains = CONFIG.alternative_domains + end + end + + def stats(env) + env.response.content_type = "application/json" + Stats.new.to_json + end + + def sharex_config(env) + host = env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]? + scheme = env.request.headers["X-Forwarded-Proto"]? || "http" + env.response.content_type = "application/json" + # So it's able to download the file instead of displaying it + env.response.headers["Content-Disposition"] = "attachment; filename=\"#{host}.sxcu\"" + + return %({ +"Version": "14.0.1", +"DestinationType": "ImageUploader, FileUploader", +"RequestMethod": "POST", +"RequestURL": "#{scheme}://#{host}/upload", +"Body": "MultipartFormData", +"FileFormName": "file", +"URL": "{json:link}", +"DeletionURL": "{json:deleteLink}", +"ErrorMessage": "{json:error}" +}) + end + + def chatterino_config(env) + host = env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]? + scheme = env.request.headers["X-Forwarded-Proto"]? || "http" + env.response.content_type = "application/json" + + return %({ +"requestUrl": "#{scheme}://#{host}/upload", +formField": "data", +imageLink": "{link}", +deleteLink": "{deleteLink}" +}) + end +end diff --git a/src/routes/retrieve.cr b/src/routes/retrieve.cr new file mode 100644 index 0000000..144c8a4 --- /dev/null +++ b/src/routes/retrieve.cr @@ -0,0 +1,56 @@ +module Routes::Retrieve + extend self + + def retrieve_file(env) + host = env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]? + scheme = env.request.headers["X-Forwarded-Proto"]? || "http" + ip_addr = env.request.headers["X-Real-IP"]? || env.request.remote_address + filename = env.params.url["filename"].split(".").first + + begin + file = Database::Files.select(filename) + if file.nil? + ee 404, "File '#{filename}' does not exist" + end + rescue ex + LOGGER.debug "Error when retrieving file '#{filename}': #{ex.message}" + ee 500, "Error when retrieving file '#{filename}'" + end + + env.response.headers["Content-Disposition"] = "inline; filename*=UTF-8''#{file.original_filename}" + env.response.headers["ETag"] = "#{file.checksum}" + + CONFIG.opengraphUseragents.each do |useragent| + env.response.content_type = "text/html" + + if env.request.headers.["User-Agent"]?.try &.includes?(useragent) + return %( + + + + + + + #{if file.thumbnail + %() + end} + + +) + end + end + send_file env, "#{CONFIG.files}/#{file.filename}#{file.extension}" + end + + def retrieve_thumbnail(env) + thumbnail = env.params.url["thumbnail"]? + pp "#{CONFIG.thumbnails}/#{thumbnail}" + + begin + send_file env, "#{CONFIG.thumbnails}/#{thumbnail}" + rescue ex + LOGGER.debug "Thumbnail '#{thumbnail}' does not exist: #{ex.message}" + ee 403, "Thumbnail '#{thumbnail}' does not exist" + end + end +end diff --git a/src/routes/upload.cr b/src/routes/upload.cr new file mode 100644 index 0000000..c5de027 --- /dev/null +++ b/src/routes/upload.cr @@ -0,0 +1,279 @@ +module Routes::Upload + extend self + + struct Response + include JSON::Serializable + + property link : String + @[JSON::Field(key: "linkExt")] + property link_ext : String + property id : String + property ext : String + property name : String + property checksum : String + @[JSON::Field(key: "deleteKey")] + property delete_key : String + @[JSON::Field(key: "deleteLink")] + property delete_link : String + + def initialize(file : UFile, scheme : String, host : String?) + @link = "#{scheme}://#{host}/#{file.filename}" + @link_ext = "#{scheme}://#{host}/#{file.filename}#{file.extension}" + @id = file.filename + @ext = file.extension + @name = file.original_filename + @checksum = file.checksum + @delete_key = file.delete_key + @delete_link = "#{scheme}://#{host}/delete?key=#{file.delete_key}" + end + end + + def upload(env) + host = env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]? + scheme = env.request.headers["X-Forwarded-Proto"]? || "http" + ip_addr = env.request.headers["X-Real-IP"]? || env.request.remote_address + env.response.content_type = "application/json" + + # You can modify this if you want to allow files smaller than 1MiB. + # This is generally a good way to check the filesize but there is a better way to do it + # which is inspecting the file directly (If I'm not wrong). + if CONFIG.size_limit > 0 + if !env.request.headers["Content-Length"]?.try &.to_i == nil + if env.request.headers["Content-Length"].to_i > 1048576*CONFIG.size_limit + ee 413, "File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB" + end + end + end + + file = UFile.new + + HTTP::FormData.parse(env.request) do |upload| + upload_filename = upload.filename + + if upload_filename + file.original_filename = upload_filename + else + LOGGER.debug "No file provided by the user" + ee 403, "No file provided" + end + + file.extension = File.extname("#{upload.filename}") + file.filename = Utils.generate_filename + full_filename = file.filename + file.extension + file_path = "#{CONFIG.files}/#{full_filename}" + + if CONFIG.blockedExtensions.includes?(file.extension.split(".")[1]) + ee 401, "Extension '#{file.extension}' is not allowed" + end + + File.open(file_path, "w") do |output| + IO.copy(upload.body, output) + end + + file.uploaded_at = Time.utc.to_unix.to_s + file.checksum = Utils::Hashing.hash_file(file_path) + end + + if CONFIG.deleteKeyLength > 0 + file.delete_key = Random.base58(CONFIG.deleteKeyLength) + end + + # X-Real-IP if behind a reverse proxy and the header is set in the reverse + # proxy configuration. + begin + spawn { Utils.generate_thumbnail(file.filename, file.extension) } + rescue ex + LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" + end + + begin + Database::Files.insert(file) + # Database::IP.insert(ip_addr) + # SQL.exec "INSERT OR IGNORE INTO ips (ip, date) VALUES (?, ?)", ip_address, Time.utc.to_unix + # # SQL.exec "INSERT OR IGNORE INTO ips (ip) VALUES ('#{ip_address}')" + # SQL.exec "UPDATE ips SET count = count + 1 WHERE ip = ('#{ip_address}')" + rescue ex + LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" + ee 500, "An error ocurred when trying to insert the data into the DB" + end + + res = Response.new(file, scheme, host) + res.to_json + end + + # The most unoptimized and unstable feature lol + # def upload_url_bulk(env) + # env.response.content_type = "application/json" + # ip_address = Utils.ip_address(env) + # protocol = Utils.protocol(env) + # host = Utils.host(env) + # begin + # files = env.params.json["files"].as((Array(JSON::Any))) + # rescue ex : JSON::ParseException + # LOGGER.error "Body malformed: #{ex.message}" + # ee 400, "Body malformed: #{ex.message}" + # rescue ex + # LOGGER.error "Unknown error: #{ex.message}" + # ee 500, "Unknown error" + # end + # successfull_files = [] of NamedTuple(filename: String, extension: String, original_filename: String, checksum: String, delete_key: String | Nil) + # failed_files = [] of String + # # X-Real-IP if behind a reverse proxy and the header is set in the reverse + # # proxy configuration. + # files.each do |url| + # url = url.to_s + # filename = Utils.generate_filename + # original_filename = "" + # extension = "" + # checksum = "" + # uploaded_at = Time.utc + # extension = File.extname(URI.parse(url).path) + # if CONFIG.deleteKeyLength > 0 + # delete_key = Random.base58(CONFIG.deleteKeyLength) + # end + # file_path = "#{CONFIG.files}/#{filename}#{extension}" + # File.open(file_path, "w") do |output| + # begin + # HTTP::Client.get(url) do |res| + # IO.copy(res.body_io, output) + # end + # rescue ex + # LOGGER.debug "Failed to download file '#{url}': #{ex.message}" + # ee 403, "Failed to download file '#{url}'" + # failed_files << url + # end + # end + # # successfull_files << url + # # end + # if extension.empty? + # extension = Utils.detect_extension(file_path) + # File.rename(file_path, file_path + extension) + # file_path = "#{CONFIG.files}/#{filename}#{extension}" + # end + # # The second one is faster and it uses less memory + # # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last + # original_filename = url.split("/").last + # checksum = Utils::Hashing.hash_file(file_path) + # begin + # spawn { Utils.generate_thumbnail(filename, extension) } + # rescue ex + # LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" + # end + # begin + # # Insert SQL data just before returning the upload information + # SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + # original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil) + # successfull_files << {filename: filename, + # original_filename: original_filename, + # extension: extension, + # delete_key: delete_key, + # checksum: checksum} + # rescue ex + # LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" + # ee 500, "An error ocurred when trying to insert the data into the DB" + # end + # end + # json = JSON.build do |j| + # j.array do + # successfull_files.each do |fileinfo| + # j.object do + # j.field "link", "#{protocol}://#{host}/#{fileinfo[:filename]}" + # j.field "linkExt", "#{protocol}://#{host}/#{fileinfo[:filename]}#{fileinfo[:extension]}" + # j.field "id", fileinfo[:filename] + # j.field "ext", fileinfo[:extension] + # j.field "name", fileinfo[:original_filename] + # j.field "checksum", fileinfo[:checksum] + # if CONFIG.deleteKeyLength > 0 + # delete_key = Random.base58(CONFIG.deleteKeyLength) + # j.field "deleteKey", fileinfo[:delete_key] + # j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{fileinfo[:delete_key]}" + # end + # end + # end + # end + # end + # json + # end + + # def upload_url(env) + # env.response.content_type = "application/json" + # ip_address = Utils.ip_address(env) + # protocol = Utils.protocol(env) + # host = Utils.host(env) + # url = env.params.query["url"] + # successfull_files = [] of NamedTuple(filename: String, extension: String, original_filename: String, checksum: String, delete_key: String | Nil) + # failed_files = [] of String + # # X-Real-IP if behind a reverse proxy and the header is set in the reverse + # # proxy configuration. + # filename = Utils.generate_filename + # original_filename = "" + # extension = "" + # checksum = "" + # uploaded_at = Time.utc + # extension = File.extname(URI.parse(url).path) + # if CONFIG.deleteKeyLength > 0 + # delete_key = Random.base58(CONFIG.deleteKeyLength) + # end + # file_path = "#{CONFIG.files}/#{filename}#{extension}" + # File.open(file_path, "w") do |output| + # begin + # # TODO: Connect timeout to prevent possible Denial of Service to the external website spamming requests + # # https://crystal-lang.org/api/1.13.2/HTTP/Client.html#connect_timeout + # HTTP::Client.get(url) do |res| + # IO.copy(res.body_io, output) + # end + # rescue ex + # LOGGER.debug "Failed to download file '#{url}': #{ex.message}" + # ee 403, "Failed to download file '#{url}': #{ex.message}" + # failed_files << url + # end + # end + # if extension.empty? + # extension = Utils.detect_extension(file_path) + # File.rename(file_path, file_path + extension) + # file_path = "#{CONFIG.files}/#{filename}#{extension}" + # end + # # The second one is faster and it uses less memory + # # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last + # original_filename = url.split("/").last + # checksum = Utils::Hashing.hash_file(file_path) + # begin + # spawn { Utils.generate_thumbnail(filename, extension) } + # rescue ex + # LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" + # end + # begin + # # Insert SQL data just before returning the upload information + # SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + # original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil) + # successfull_files << {filename: filename, + # original_filename: original_filename, + # extension: extension, + # delete_key: delete_key, + # checksum: checksum} + # rescue ex + # LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" + # ee 500, "An error ocurred when trying to insert the data into the DB" + # end + # json = JSON.build do |j| + # j.array do + # successfull_files.each do |fileinfo| + # j.object do + # j.field "link", "#{protocol}://#{host}/#{fileinfo[:filename]}" + # j.field "linkExt", "#{protocol}://#{host}/#{fileinfo[:filename]}#{fileinfo[:extension]}" + # j.field "id", fileinfo[:filename] + # j.field "ext", fileinfo[:extension] + # j.field "name", fileinfo[:original_filename] + # j.field "checksum", fileinfo[:checksum] + # if CONFIG.deleteKeyLength > 0 + # delete_key = Random.base58(CONFIG.deleteKeyLength) + # j.field "deleteKey", fileinfo[:delete_key] + # j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{fileinfo[:delete_key]}" + # end + # end + # end + # end + # end + # json + # end +end diff --git a/src/routes/views.cr b/src/routes/views.cr new file mode 100644 index 0000000..4b025e3 --- /dev/null +++ b/src/routes/views.cr @@ -0,0 +1,18 @@ +module Routes::Views + extend self + + def root(env) + host = env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]? + scheme = env.request.headers["X-Forwarded-Proto"]? || "http" + files_hosted = Database::Files.file_count + + render "src/views/index.ecr" + end + + def chatterino(env) + host = env.request.headers["X-Forwarded-Host"]? || env.request.headers["Host"]? + scheme = env.request.headers["X-Forwarded-Proto"]? || "http" + + render "src/views/chatterino.ecr" + end +end diff --git a/src/routing.cr b/src/routing.cr index 6e8f8dd..27dadf6 100644 --- a/src/routing.cr +++ b/src/routing.cr @@ -1,26 +1,50 @@ -require "./http-errors" +require "./macros" +require "./routes/**" module Routing extend self - @@exit_nodes = Array(String).new - def reload_exit_nodes - LOGGER.debug "Updating Tor exit nodes array" - @@exit_nodes = Utils.load_tor_exit_nodes - LOGGER.debug "IPs inside the Tor exit nodes array: #{@@exit_nodes.size}" - end + {% for http_method in {"get", "post", "delete", "options", "patch", "put"} %} - before_post "/api/admin/*" do |env| - if env.request.headers.try &.["X-Api-Key"]? != CONFIG.adminApiKey || nil - halt env, status_code: 401, response: http_error 401, "Wrong API Key" + macro {{http_method.id}}(path, controller, method = :handle) + unless Kemal::Utils.path_starts_with_slash?(\{{path}}) + raise Kemal::Exceptions::InvalidPathStartException.new({{http_method}}, \{{path}}) + end + + Kemal::RouteHandler::INSTANCE.add_route({{http_method.upcase}}, \{{path}}) do |env| + \{{ controller }}.\{{ method.id }}(env) + end + end + + {% end %} + + # before_post "/api/admin/*" do |env| + # env.response.content_type = "application/json" + + # if env.request.headers.try &.["X-Api-Key"]? != CONFIG.adminApiKey || nil + # halt env, status_code: 401, response: "Wrong API Key" + # end + # end + + before_post do |env| + tor_exit_nodes = Utils::Tor.exit_nodes + api_key = env.request.headers["X-Api-Key"]? + + # Skips Tor blocking and Rate limits if the API key matches + if api_key == CONFIG.adminApiKey + next + end + + if CONFIG.blockTorAddresses && tor_exit_nodes.includes?(Headers.ip_addr) + halt env, status_code: 401, response: CONFIG.torMessage end end before_post "/upload" do |env| begin - ip_info = SQL.query_one?("SELECT ip, count, date FROM ips WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32}) + ip_info = SQL.query_one?("SELECT ip, count, date FROM ips WHERE ip = ?", Headers.ip_addr, as: {ip: String, count: Int32, date: Int32}) rescue ex - LOGGER.error "Error when trying to enforce rate limits: #{ex.message}" + LOGGER.error "Error when trying to enforce rate limits for ip #{Headers.ip_addr}: #{ex.message}" next end @@ -35,97 +59,56 @@ module Routing end if CONFIG.filesPerIP > 0 if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod - halt env, status_code: 401, response: http_error 401, "Rate limited! Try again in #{time_until_unban} seconds" + halt env, status_code: 401, response: "Rate limited! Try again in #{time_until_unban} seconds" end end end - before_post do |env| - if env.request.headers.try &.["X-Api-Key"]? == CONFIG.adminApiKey - # Skips Tor and Rate limits if the API key matches - next - end - if CONFIG.blockTorAddresses && @@exit_nodes.includes?(Utils.ip_address(env)) - halt env, status_code: 401, response: http_error 401, CONFIG.torMessage - end - end - def register_all - get "/" do |env| - host = Utils.host(env) - files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 - render "src/views/index.ecr" - end + get "/", Routes::Views, :root + get "/info/chatterino", Routes::Views, :chatterino - get "/chatterino" do |env| - host = Utils.host(env) - protocol = Utils.protocol(env) - render "src/views/chatterino.ecr" - end + post "/upload", Routes::Upload, :upload + # get "/upload", Routes::Upload, :upload_url + # post "/api/uploadurl", Routes::Upload, :upload_url - post "/upload" do |env| - Handling.upload(env) - end + get "/:filename", Routes::Retrieve, :retrieve_file + get "/thumbnail/:thumbnail", Routes::Retrieve, :retrieve_thumbnail - get "/upload" do |env| - Handling.upload_url(env) - end + get "/delete", Routes::Deletion, :delete_file - post "/api/uploadurl" do |env| - Handling.upload_url_bulk(env) - end + get "/api/stats", Routing::Misc, :stats + get "/info/sharex.sxcu", Routing::Misc, :sharex_config + get "/info/chatterinoconfig", Routing::Misc, :chatterino_config - get "/:filename" do |env| - Handling.retrieve_file(env) - end - - get "/thumbnail/:thumbnail" do |env| - Handling.retrieve_thumbnail(env) - end - - get "/delete" do |env| - Handling.delete_file(env) - end - - get "/api/stats" do |env| - Handling.stats(env) - end - - get "/sharex.sxcu" do |env| - Handling.sharex_config(env) - end - - get "/chatterinoconfig" do |env| - Handling.chatterino_config(env) - end - - if CONFIG.adminEnabled - self.register_admin - end + # if CONFIG.adminEnabled + # self.register_admin + # end end - def register_admin - # post "/api/admin/upload" do |env| - # Handling::Admin.delete_ip_limit(env) - # end - post "/api/admin/delete" do |env| - Handling::Admin.delete_file(env) - end - end + # def register_admin + # # post "/api/admin/upload" do |env| + # # Routes::Admin.delete_ip_limit(env) + # # end + # post "/api/admin/delete" do |env| + # Routes::Admin.delete_file(env) + # end + # end - post "/api/admin/deleteiplimit" do |env| - Handling::Admin.delete_ip_limit(env) - end + # post "/api/admin/deleteiplimit" do |env| + # Routes::Admin.delete_ip_limit(env) + # end - post "/api/admin/fileinfo" do |env| - Handling::Admin.retrieve_file_info(env) - end + # post "/api/admin/fileinfo" do |env| + # Routes::Admin.retrieve_file_info(env) + # end - get "/api/admin/torexitnodes" do |env| - Handling::Admin.retrieve_tor_exit_nodes(env, @@exit_nodes) - end + # get "/api/admin/torexitnodes" do |env| + # Routes::Admin.retrieve_tor_exit_nodes(env, @@exit_nodes) + # end - error 404 do - "File not found" + error 404 do |env| + env.response.content_type = "text/plain" + "File not found.\nArchivo no encontrado." end end diff --git a/src/types/ip.cr b/src/types/ip.cr new file mode 100644 index 0000000..d1cd187 --- /dev/null +++ b/src/types/ip.cr @@ -0,0 +1,24 @@ +struct IP + # Without this, this class will not be able to be used as `as: UFile` on + # SQL queries + include DB::Serializable + + property ip : String + property count : Int32 + property unix_date : Int32 + + def initialize( + @ip, + @count, + @unix_date, + ) + end + + def to_tuple + {% begin %} + { + {{@type.instance_vars.map(&.name).splat}} + } + {% end %} + end +end diff --git a/src/types/ufile.cr b/src/types/ufile.cr new file mode 100644 index 0000000..d35c705 --- /dev/null +++ b/src/types/ufile.cr @@ -0,0 +1,34 @@ +struct UFile + # Without this, this class will not be able to be used as `as: UFile` on + # SQL queries + include DB::Serializable + + property original_filename : String = "" + property filename : String = "" + property extension : String = "" + property uploaded_at : String = "" + property checksum : String = "" + property ip : String = "" + property delete_key : String = "" + property thumbnail : String? + + def initialize( + @original_filename = "", + @filename = "", + @extension = "", + @uploaded_at = "", + @checksum = "", + @ip = "", + @delete_key = "", + @thumbnail = nil, + ) + end + + def to_tuple + {% begin %} + { + {{@type.instance_vars.map(&.name).splat}} + } + {% end %} + end +end diff --git a/src/utils/hashing.cr b/src/utils/hashing.cr new file mode 100644 index 0000000..8eb096a --- /dev/null +++ b/src/utils/hashing.cr @@ -0,0 +1,11 @@ +module Utils::Hashing + extend self + + def hash_file(file_path : String) : String + Digest::SHA1.hexdigest &.file(file_path) + end + + def hash_io(file_path : IO) : String + Digest::SHA1.hexdigest &.update(file_path) + end +end diff --git a/src/utils/tor.cr b/src/utils/tor.cr new file mode 100644 index 0000000..b266d08 --- /dev/null +++ b/src/utils/tor.cr @@ -0,0 +1,38 @@ +module Utils::Tor + extend self + @@exit_nodes : Array(String) = [] of String + + def refresh_exit_nodes + LOGGER.debug "reload_exit_nodes: Updating Tor exit nodes list" + retrieve_tor_exit_nodes + LOGGER.debug "reload_exit_nodes: IPs inside the Tor exit nodes list: #{@@exit_nodes.size}" + end + + def retrieve_tor_exit_nodes + LOGGER.debug "retrieve_tor_exit_nodes: Retrieving Tor exit nodes list" + ips = [] of String + + HTTP::Client.get(CONFIG.torExitNodesUrl) do |res| + begin + if res.success? && res.status_code == 200 + res.body_io.each_line do |line| + if line.includes?("ExitAddress") + ips << line.split(" ")[1] + end + end + @@exit_nodes = ips + else + LOGGER.error "retrieve_tor_exit_nodes: Failed to retrieve exit nodes list. Status Code: #{res.status_code}" + end + rescue ex : Socket::ConnectError + LOGGER.error "retrieve_tor_exit_nodes: Failed to connect to #{CONFIG.torExitNodesUrl}: #{ex.message}" + rescue ex + LOGGER.error "retrieve_tor_exit_nodes: Unknown error: #{ex.message}" + end + end + end + + def exit_nodes : Array(String) + return @@exit_nodes + end +end diff --git a/src/utils.cr b/src/utils/utils.cr similarity index 50% rename from src/utils.cr rename to src/utils/utils.cr index 083146e..d76f4aa 100644 --- a/src/utils.cr +++ b/src/utils/utils.cr @@ -43,25 +43,57 @@ module Utils end end - def check_old_files - LOGGER.info "Deleting old files" - fileinfo = SQL.query_all("SELECT filename, extension, thumbnail - FROM files - WHERE uploaded_at < datetime('now', '-#{CONFIG.deleteFilesAfter} days')", - as: {filename: String, extension: String, thumbnail: String | Nil}) + def delete_file(env) + key = env.params.query["key"] + file = SQL.select_with_key(key) + full_filename = file.filename + file.extension + thumbnail = file.thumbnail - fileinfo.each do |file| - LOGGER.debug "Deleting file '#{file[:filename]}#{file[:extension]}'" + # Delete file + File.delete("#{CONFIG.files}/#{full_filename}") + + if file.thumbnail + File.delete("#{CONFIG.thumbnails}/#{thumbnail}") + end + + # Delete entry from db + Database::Files.delete_with_key(key) + + LOGGER.debug "File '#{full_filename}' was deleted using key '#{key}'}" + msg("File '#{full_filename}' deleted successfully") + end + + # TODO: Spawn a fiber and add each file to an array to bulk delete files from + # the database using a single SQL query. + # In the end, all old files should be not accessible, even if they are on the + # drive. + def check_old_files + LOGGER.info "check_old_files: Deleting old files" + files = Database::Files.old_files + + files.each do |f| + full_filename = f.filename + f.extension + thumbnail = f.thumbnail + + # TODO: Check if it's able to bypass the path using a filename with a `/` in their name + LOGGER.debug "check_old_files: Deleting file '#{full_filename}'" begin - File.delete("#{CONFIG.files}/#{file[:filename]}#{file[:extension]}") - if file[:thumbnail] - File.delete("#{CONFIG.thumbnails}/#{file[:thumbnail]}") + File.delete("#{CONFIG.files}/#{full_filename}") + + if thumbnail + File.delete("#{CONFIG.thumbnails}/#{thumbnail}") end - SQL.exec "DELETE FROM files WHERE filename = ?", file[:filename] + + Database::Files.delete(f.filename) + rescue File::NotFoundError + LOGGER.error "check_old_files: File '#{full_filename}' doesn't seem to exist on the '#{CONFIG.files}', folder, deleting it from the database" + Database::Files.delete(f.filename) + rescue ex : File::AccessDeniedError + LOGGER.error "check_old_files: File '#{full_filename}' failed to be deleted due to bad permissions, deleting it from the database: #{ex.message}" + Database::Files.delete(f.filename) rescue ex - LOGGER.error "#{ex.message}" - # Also delete the file entry from the DB if it doesn't exist. - SQL.exec "DELETE FROM files WHERE filename = ?", file[:filename] + LOGGER.error "check_old_files: File '#{full_filename}' failed to be deleted, deleting it from the database: #{ex.message}" + Database::Files.delete(f.filename) end end end @@ -69,7 +101,7 @@ module Utils def check_dependencies dependencies = ["ffmpeg"] dependencies.each do |dep| - next if !CONFIG.generateThumbnails + next if !CONFIG.generate_thumbnails if !Process.find_executable(dep) LOGGER.fatal("'#{dep}' was not found.") exit(1) @@ -77,39 +109,17 @@ module Utils end end - # TODO: - # def check_duplicate(upload) - # file_checksum = SQL.query_all("SELECT checksum FROM files WHERE original_filename = ?", upload.filename, as:String).try &.[0]? - # if file_checksum.nil? - # return - # else - # uploaded_file_checksum = hash_io(upload.body) - # pp file_checksum - # pp uploaded_file_checksum - # if file_checksum == uploaded_file_checksum - # puts "Dupl" - # end - # end - # end - - def hash_file(file_path : String) : String - Digest::SHA1.hexdigest &.file(file_path) - end - - def hash_io(file_path : IO) : String - Digest::SHA1.hexdigest &.update(file_path) - end - # TODO: Check if there are no other possibilities to get a random filename and exit def generate_filename - filename = Random.base58(CONFIG.fileameLength) + filename = Random.base58(CONFIG.filename_length) loop do - if SQL.query_one("SELECT COUNT(filename) FROM files WHERE filename = ?", filename, as: Int32) == 0 + file = Database::Files.select(filename) + if !file return filename else - LOGGER.debug "Filename collision! Generating a new filename" - filename = Random.base58(CONFIG.fileameLength) + LOGGER.trace "Filename collision! Generating a new filename" + filename = Random.base58(CONFIG.filename_length) end end end @@ -117,13 +127,15 @@ module Utils def generate_thumbnail(filename, extension) exts = [".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".webp", ".heic", ".jxl", ".avif", ".crw", ".dng", ".mp4", ".mkv", ".webm", ".avi", ".wmv", ".flv", "m4v", ".mov", ".amv", ".3gp", ".mpg", ".mpeg", ".yuv"] - # To prevent thumbnail generation on non image extensions - return if exts.none? do |ext| - extension.downcase.includes?(ext) - end + + # To prevent thumbnail generation on non image extensions + return if exts.none? { |ext| extension.downcase.includes?(ext) } + # Disable generation if false - return if !CONFIG.generateThumbnails || !CONFIG.thumbnails + return if !CONFIG.generate_thumbnails || !CONFIG.thumbnails + LOGGER.debug "Generating thumbnail for #{filename + extension} in background" + process = Process.run("ffmpeg", [ "-hide_banner", @@ -137,11 +149,12 @@ module Utils "-update", "1", "#{CONFIG.thumbnails}/#{filename}.jpg", ]) - if process.exit_code == 0 - LOGGER.debug "Thumbnail for #{filename + extension} generated successfully" + + if process.normal_exit? + LOGGER.debug "Thumbnail for '#{filename + extension}' generated successfully" SQL.exec "UPDATE files SET thumbnail = ? WHERE filename = ?", filename + ".jpg", filename else - # TODO: Add some sort of message when the thumbnail is not generated + LOGGER.debug "Failed to generate thumbnail for '#{filename + extension}'. Exit code of ffmpeg: #{process.exit_code}" end end @@ -159,25 +172,6 @@ module Utils end end - def delete_file(env) - fileinfo = SQL.query_all("SELECT filename, extension, thumbnail - FROM #{CONFIG.dbTableName} - WHERE delete_key = ?", - env.params.query["key"], - as: {filename: String, extension: String, thumbnail: String | Nil})[0] - - # Delete file - File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}") - if fileinfo[:thumbnail] - File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") - end - # Delete entry from db - SQL.exec "DELETE FROM files WHERE delete_key = ?", env.params.query["key"] - - LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" - msg("File '#{fileinfo[:filename]}' deleted successfully") - end - MAGIC_BYTES = { # Images ".png" => "89504e470d0a1a0a", @@ -221,60 +215,4 @@ module Utils end "" end - - def retrieve_tor_exit_nodes - LOGGER.debug "Retrieving Tor exit nodes list" - HTTP::Client.get(CONFIG.torExitNodesUrl) do |res| - begin - if res.success? && res.status_code == 200 - begin - File.open(CONFIG.torExitNodesFile, "w") { |output| IO.copy(res.body_io, output) } - rescue ex - LOGGER.error "Failed to save exit nodes list: #{ex.message}" - end - else - LOGGER.error "Failed to retrieve exit nodes list. Status Code: #{res.status_code}" - end - rescue ex : Socket::ConnectError - LOGGER.error "Failed to connect to #{CONFIG.torExitNodesUrl}: #{ex.message}" - rescue ex - LOGGER.error "Unknown error: #{ex.message}" - end - end - end - - def load_tor_exit_nodes - exit_nodes = File.read_lines(CONFIG.torExitNodesFile) - ips = [] of String - exit_nodes.each do |line| - if line.includes?("ExitAddress") - ips << line.split(" ")[1] - end - end - return ips - end - - def ip_address(env) : String - begin - return env.request.headers.try &.["X-Forwarded-For"] - rescue - return env.request.remote_address.to_s.split(":").first - end - end - - def protocol(env) : String - begin - return env.request.headers.try &.["X-Forwarded-Proto"] - rescue - return "http" - end - end - - def host(env) : String - begin - return env.request.headers.try &.["X-Forwarded-Host"] - rescue - return env.request.headers["Host"] - end - end end diff --git a/src/views/chatterino.ecr b/src/views/chatterino.ecr index b8dc463..d7572ac 100644 --- a/src/views/chatterino.ecr +++ b/src/views/chatterino.ecr @@ -4,17 +4,18 @@ <%= host %> - - + +
-

Chatterino config

-

Request URL: <%= protocol %>://<%= host %>/upload

-

Form field: data

-

Image link: link

-

Delete link: deleteLink

+

Chatterino Config

+

Request URL: <%= scheme %>://<%= host %>/upload

+

Form field: data

+

Image link: link

+

Delete link: deleteLink

+
diff --git a/src/views/index.ecr b/src/views/index.ecr index dd8d711..9b691c9 100644 --- a/src/views/index.ecr +++ b/src/views/index.ecr @@ -4,8 +4,8 @@ <%= host %> - - + + @@ -21,24 +21,22 @@
-

- Chatterino Config | - ShareX Config | - - file-uploader-crystal (BETA <%= CURRENT_TAG %> - <%= CURRENT_VERSION %> @ <%= CURRENT_BRANCH %>) - -

-

Archivos alojados: <%= files_hosted %>

- <% if CONFIG.blockTorAddresses %> -

<%= CONFIG.torMessage %>

- <% end %> - <% if !CONFIG.alternativeDomains.empty? %> -

- <% CONFIG.alternativeDomains.each do | domain | %> - <%= domain %> - <% end %> -

- <% end %> +

+ Chatterino Config | + ShareX Config | + + file-uploader-crystal (BETA <%= CURRENT_TAG %> - <%= CURRENT_VERSION %> @ <%= CURRENT_BRANCH %>) + +

+

Archivos alojados: <%= files_hosted %>

+ <% if !CONFIG.alternative_domains.empty? %> +

+ <% CONFIG.alternative_domains.each do | domain | %> + <%= domain %> + <% end %> +

+ <% end %>
+