From 9cd9bc689ab70cf2942705957e91ed5f06b1ae1b Mon Sep 17 00:00:00 2001 From: Fijxu Date: Sat, 10 Aug 2024 21:13:37 -0400 Subject: [PATCH] 0.8.2: Tor exit node blocking. --- .gitignore | 2 + README.md | 8 +- config/config.yml | 22 ++-- src/config.cr | 30 +++-- src/file-uploader.cr | 2 +- src/handling.cr | 268 --------------------------------------- src/handling/admin.cr | 6 +- src/handling/handling.cr | 30 ++--- src/http-errors.cr | 12 +- src/jobs.cr | 17 ++- src/routing.cr | 26 +++- src/utils.cr | 71 +++++++---- src/views/index.ecr | 3 + 13 files changed, 149 insertions(+), 348 deletions(-) delete mode 100644 src/handling.cr diff --git a/.gitignore b/.gitignore index 0bb75ea..d72439e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ /bin/ /.shards/ *.dwarf +data +torexitnodes.txt diff --git a/README.md b/README.md index 96d6c13..fac69fc 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,11 @@ WantedBy=default.target - ~~Add file size limit~~ ADDED - ~~Fix error when accessing `http://127.0.0.1:8080` with an empty DB.~~ Fixed somehow. - Better frontend... -- ~~Disable file deletion if `delete_files_after_check_seconds` or `delete_files_after` is set to `0`~~ DONE -- ~~Disable delete key if `delete_key_length` is `0`~~ DONE (But I think there is a better way to do it) -- ~~Exit if `filename_length` is `0`~~ DONE +- ~~Disable file deletion if `deleteFilesCheck` or `deleteFilesAfter` is set to `0`~~ DONE +- ~~Disable delete key if `deleteKeyLength` is `0`~~ DONE (But I think there is a better way to do it) +- ~~Exit if `fileameLength` is `0`~~ DONE - ~~Disable file limit if `size_limit` is `0`~~ DONE - ~~Prevent files from being overwritten in the event of a name collision~~ DONE - Dockerfile and Docker image (Crystal doesn't has dependency hell like other languages so is not really necessary to do, but useful for people that want instant deploy) +- Custom file expiration using headers (Like rustypaste) +- Small CLI to upload files (like `rpaste` from rustypaste) diff --git a/config/config.yml b/config/config.yml index 4ec2611..8f4ee4f 100644 --- a/config/config.yml +++ b/config/config.yml @@ -1,30 +1,36 @@ files: "./files" thumbnails: "./thumbnails" -generate_thumbnails: false +generateThumbnails: false db: "./db.sqlite3" -db_table_name: "files" +dbTableName: "files" adminEnabled: false adminApiKey: "asd" -filename_length: 3 +fileameLength: 3 # In MiB size_limit: 512 port: 8080 +blockTorAddresses: true +# Every hour +torExitNodesCheck: 3600 +torExitNodesUrl: "https://www.dan.me.uk/torlist/?exit" +torExitNodesFile: "./torexitnodes.txt" +torMessage: "TOR IS BLOCKED!" # If you define the unix socket, it will only listen on the socket and not the port. #unix_socket: "/tmp/file-uploader.sock" # In days -delete_files_after: 7 +deleteFilesAfter: 7 # In seconds -delete_files_after_check_seconds: 1800 -delete_key_length: 4 +deleteFilesCheck: 1800 +deleteKeyLength: 4 siteInfo: "Whatever you want to put here" siteWarning: "WARNING!" log_level: "debug" -blocked_extensions: +blockedExtensions: - "exe" # List of useragents that use OpenGraph to gather file information -opengraph_useragents: +opengraphUseragents: - "chatterino-api-cache/" - "FFZBot/" - "Twitterbot/" diff --git a/src/config.cr b/src/config.cr index 674e556..3aa5027 100644 --- a/src/config.cr +++ b/src/config.cr @@ -5,24 +5,30 @@ class Config property files : String = "./files" property thumbnails : String = "./thumbnails" - property generate_thumbnails : Bool = false + property generateThumbnails : Bool = false property db : String = "./db.sqlite3" - property db_table_name : String = "files" + property dbTableName : String = "files" property adminEnabled : Bool = false - property adminApiKey : String = "" - property incremental_filename_length : Bool = true - property filename_length : Int32 = 3 + property adminApiKey : String? = "" + property incremental_fileameLength : Bool = true + property fileameLength : Int32 = 3 # In MiB property size_limit : Int16 = 512 property port : Int32 = 8080 property unix_socket : String? - property delete_files_after : Int32 = 7 + property blockTorAddresses : Bool? = false + property torExitNodesCheck : Int32 = 3600 + # The list needs to contain a IP address per line + property torExitNodesUrl : String = "https://www.dan.me.uk/torlist/?exit" + property torExitNodesFile : String = "./torexitnodes.txt" + property torMessage : String? = "Tor is blocked!" + property deleteFilesAfter : Int32 = 7 # How often should the check of old files be performed? (in seconds) - property delete_files_after_check_seconds : Int32 = 1800 - property delete_key_length : Int32 = 4 + property deleteFilesCheck : Int32 = 1800 + property deleteKeyLength : Int32 = 4 # Blocked extensions that are not allowed to be uploaded to the server - property blocked_extensions : Array(String) = [] of String - property opengraph_useragents : Array(String) = [] of String + property blockedExtensions : Array(String) = [] of String + property opengraphUseragents : Array(String) = [] of String property siteInfo : String = "xd" property siteWarning : String? = "" property log_level : LogLevel = LogLevel::Info @@ -36,8 +42,8 @@ class Config end def self.check_config(config : Config) - if config.filename_length <= 0 - puts "Config: filename_length cannot be #{config.filename_length}" + if config.fileameLength <= 0 + puts "Config: fileameLength cannot be #{config.fileameLength}" exit(1) end end diff --git a/src/file-uploader.cr b/src/file-uploader.cr index 7504f59..d775fb6 100644 --- a/src/file-uploader.cr +++ b/src/file-uploader.cr @@ -20,7 +20,7 @@ Kemal.config.app_name = "file-uploader-crystal" # https://github.com/iv-org/invidious/blob/90e94d4e6cc126a8b7a091d12d7a5556bfe369d5/src/invidious.cr#L136C1-L136C61 LOGGER = LogHandler.new(STDOUT, CONFIG.log_level) # Give me a 128 bit CPU -# MAX_FILES = 58**CONFIG.filename_length +# MAX_FILES = 58**CONFIG.fileameLength SQL = DB.open("sqlite3://#{CONFIG.db}") # https://github.com/iv-org/invidious/blob/90e94d4e6cc126a8b7a091d12d7a5556bfe369d5/src/invidious.cr#L78 diff --git a/src/handling.cr b/src/handling.cr deleted file mode 100644 index 26eb14c..0000000 --- a/src/handling.cr +++ /dev/null @@ -1,268 +0,0 @@ -require "./http-errors" -require "http/client" - -module Handling - extend self - - def upload(env) - env.response.content_type = "application/json" - # You can modify this if you want to allow files smaller than 1MiB. - # This is generally a good way to check the filesize but there is a better way to do it - # which is inspecting the file directly (If I'm not wrong). - if CONFIG.size_limit > 0 - if env.request.headers["Content-Length"].to_i > 1048576*CONFIG.size_limit - error413("File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB") - end - end - filename = "" - extension = "" - original_filename = "" - uploaded_at = "" - checksum = "" - ip_address = "" - delete_key = nil - # TODO: Return the file that matches a checksum inside the database - HTTP::FormData.parse(env.request) do |upload| - if upload.filename.nil? || upload.filename.to_s.empty? - LOGGER.debug "No file provided by the user" - error403("No file provided") - end - # TODO: upload.body is emptied when is copied or read - # Utils.check_duplicate(upload.dup) - extension = File.extname("#{upload.filename}") - if CONFIG.blocked_extensions.includes?(extension.split(".")[1]) - error401("Extension '#{extension}' is not allowed") - end - filename = Utils.generate_filename - file_path = ::File.join ["#{CONFIG.files}", filename + extension] - File.open(file_path, "w") do |output| - IO.copy(upload.body, output) - end - original_filename = upload.filename - uploaded_at = Time::Format::HTTP_DATE.format(Time.utc) - checksum = Utils.hash_file(file_path) - end - protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" - host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] - # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse - # proxy configuration. - ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first - json = JSON.build do |j| - j.object do - j.field "link", "#{protocol}://#{host}/#{filename}" - j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}" - j.field "id", filename - j.field "ext", extension - j.field "name", original_filename - j.field "checksum", checksum - if CONFIG.delete_key_length > 0 - delete_key = Random.base58(CONFIG.delete_key_length) - j.field "deleteKey", delete_key - j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}" - end - end - end - begin - spawn { Utils.generate_thumbnail(filename, extension) } - rescue ex - LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" - end - begin - # Insert SQL data just before returning the upload information - SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil - rescue ex - LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" - error500("An error ocurred when trying to insert the data into the DB") - end - return json - end - - # The most unoptimized and unstable feature lol - def upload_url(env) - env.response.content_type = "application/json" - extension = "" - filename = Utils.generate_filename - original_filename = "" - uploaded_at = Time::Format::HTTP_DATE.format(Time.utc) - checksum = "" - ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first - delete_key = nil - # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse - # proxy configuration. - if !env.params.body.nil? || env.params.body["url"].empty? - url = env.params.body["url"] - extension = File.extname(URI.parse(url).path) - file_path = ::File.join ["#{CONFIG.files}", filename + extension] - File.open(file_path, "w") do |output| - begin - HTTP::Client.get(url) do |res| - IO.copy(res.body_io, output) - end - rescue ex - LOGGER.debug "Failed to download file '#{url}': #{ex.message}" - error403("Failed to download file '#{url}'") - end - end - if extension.empty? - extension = Utils.detect_extension(file_path) - File.rename(file_path, file_path + extension) - file_path = ::File.join ["#{CONFIG.files}", filename + extension] - end - # TODO: Benchmark this: - # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last - original_filename = url.split("/").last - checksum = Utils.hash_file(file_path) - if !filename.empty? - protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" - host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] - json = JSON.build do |j| - j.object do - j.field "link", "#{protocol}://#{host}/#{filename}" - j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}" - j.field "id", filename - j.field "ext", extension - j.field "name", original_filename - j.field "checksum", checksum - if CONFIG.delete_key_length > 0 - delete_key = Random.base58(CONFIG.delete_key_length) - j.field "deleteKey", delete_key - j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}" - end - end - end - begin - spawn { Utils.generate_thumbnail(filename, extension) } - rescue ex - LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}" - end - begin - # Insert SQL data just before returning the upload information - SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil - rescue ex - LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" - error500("An error ocurred when trying to insert the data into the DB") - end - return json - end - else - end - error403("Data malformed") - end - - def retrieve_file(env) - protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" - host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] - begin - fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail - FROM #{CONFIG.db_table_name} - WHERE filename = ?", - env.params.url["filename"].split(".").first, - as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0] - - headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"}) - headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"}) - headers(env, {"ETag" => "#{fileinfo[:checksum]}"}) - - if env.request.headers.try &.["User-Agent"].includes?("chatterino-api-cache/") || env.request.headers.try &.["User-Agent"].includes?("FFZBot/") || env.request.headers.try &.["User-Agent"].includes?("Twitterbot/") - env.response.content_type = "text/html" - return %( - - - - - - - #{if fileinfo[:thumbnail] - %() - end} - - -) - end - send_file env, "#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:ext]}" - rescue ex - LOGGER.debug "File '#{env.params.url["filename"]}' does not exist: #{ex.message}" - error403("File '#{env.params.url["filename"]}' does not exist") - end - end - - def retrieve_thumbnail(env) - begin - send_file env, "#{CONFIG.thumbnails}/#{env.params.url["thumbnail"]}" - rescue ex - LOGGER.debug "Thumbnail '#{env.params.url["thumbnail"]}' does not exist: #{ex.message}" - error403("Thumbnail '#{env.params.url["thumbnail"]}' does not exist") - end - end - - def stats(env) - env.response.content_type = "application/json" - begin - json_data = JSON.build do |json| - json.object do - json.field "stats" do - json.object do - json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.db_table_name}", as: Int32 - json.field "maxUploadSize", CONFIG.size_limit - json.field "thumbnailGeneration", CONFIG.generate_thumbnails - json.field "filenameLength", CONFIG.filename_length - end - end - end - end - rescue ex - LOGGER.error "Unknown error: #{ex.message}" - error500("Unknown error") - end - json_data - end - - def delete_file(env) - if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.db_table_name} WHERE delete_key = ?)", env.params.query["key"], as: Bool - begin - fileinfo = SQL.query_all("SELECT filename, extension, thumbnail - FROM #{CONFIG.db_table_name} - WHERE delete_key = ?", - env.params.query["key"], - as: {filename: String, extension: String, thumbnail: String | Nil})[0] - - # Delete file - File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}") - if fileinfo[:thumbnail] - # Delete thumbnail - File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") - end - # Delete entry from db - SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"] - LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" - msg("File '#{fileinfo[:filename]}' deleted successfully") - rescue ex - LOGGER.error("Unknown error: #{ex.message}") - error500("Unknown error") - end - else - LOGGER.debug "Key '#{env.params.query["key"]}' does not exist" - error401("Delete key '#{env.params.query["key"]}' does not exist. No files were deleted") - end - end - - def sharex_config(env) - protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http" - host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] - env.response.content_type = "application/json" - env.response.headers["Content-Disposition"] = "attachment; filename=\"#{host}.sxcu\"" - return %({ - "Version": "14.0.1", - "DestinationType": "ImageUploader, FileUploader", - "RequestMethod": "POST", - "RequestURL": "#{protocol}://#{host}/upload", - "Body": "MultipartFormData", - "FileFormName": "file", - "URL": "{json:link}", - "DeletionURL": "{json:deleteLink}", - "ErrorMessage": "{json:error}" -}) - end -end diff --git a/src/handling/admin.cr b/src/handling/admin.cr index c83933b..820d9b0 100644 --- a/src/handling/admin.cr +++ b/src/handling/admin.cr @@ -14,7 +14,7 @@ module Handling::Admin file = file.to_s begin fileinfo = SQL.query_one("SELECT filename, extension, thumbnail - FROM #{CONFIG.db_table_name} + FROM #{CONFIG.dbTableName} WHERE filename = ?", file, as: {filename: String, extension: String, thumbnail: String | Nil}) @@ -26,11 +26,11 @@ module Handling::Admin File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") end # Delete entry from db - SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE filename = ?", file + SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE filename = ?", file LOGGER.debug "File '#{fileinfo[:filename]}' was deleted" successfull_files << file rescue ex : DB::NoResultsError - LOGGER.error("File '#{file}' doesn't exist: #{ex.message}") + LOGGER.error("File '#{file}' doesn't exist or is not registered in the database: #{ex.message}") failed_files << file rescue ex LOGGER.error "Unknown error: #{ex.message}" diff --git a/src/handling/handling.cr b/src/handling/handling.cr index 1af6c71..d35ec7e 100644 --- a/src/handling/handling.cr +++ b/src/handling/handling.cr @@ -30,7 +30,7 @@ module Handling # TODO: upload.body is emptied when is copied or read # Utils.check_duplicate(upload.dup) extension = File.extname("#{upload.filename}") - if CONFIG.blocked_extensions.includes?(extension.split(".")[1]) + if CONFIG.blockedExtensions.includes?(extension.split(".")[1]) error401("Extension '#{extension}' is not allowed") end filename = Utils.generate_filename @@ -55,8 +55,8 @@ module Handling j.field "ext", extension j.field "name", original_filename j.field "checksum", checksum - if CONFIG.delete_key_length > 0 - delete_key = Random.base58(CONFIG.delete_key_length) + if CONFIG.deleteKeyLength > 0 + delete_key = Random.base58(CONFIG.deleteKeyLength) j.field "deleteKey", delete_key j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}" end @@ -69,7 +69,7 @@ module Handling end begin # Insert SQL data just before returning the upload information - SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + SQL.exec "INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil rescue ex LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" @@ -124,8 +124,8 @@ module Handling j.field "ext", extension j.field "name", original_filename j.field "checksum", checksum - if CONFIG.delete_key_length > 0 - delete_key = Random.base58(CONFIG.delete_key_length) + if CONFIG.deleteKeyLength > 0 + delete_key = Random.base58(CONFIG.deleteKeyLength) j.field "deleteKey", delete_key j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}" end @@ -138,7 +138,7 @@ module Handling end begin # Insert SQL data just before returning the upload information - SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + SQL.exec "INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)", original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil rescue ex LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}" @@ -156,7 +156,7 @@ module Handling host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"] begin fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail - FROM #{CONFIG.db_table_name} + FROM #{CONFIG.dbTableName} WHERE filename = ?", env.params.url["filename"].split(".").first, as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0] @@ -165,7 +165,7 @@ module Handling headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"}) headers(env, {"ETag" => "#{fileinfo[:checksum]}"}) - CONFIG.opengraph_useragents.each do |useragent| + CONFIG.opengraphUseragents.each do |useragent| if env.request.headers.try &.["User-Agent"].includes?(useragent) env.response.content_type = "text/html" return %( @@ -206,10 +206,10 @@ module Handling json.object do json.field "stats" do json.object do - json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.db_table_name}", as: Int32 + json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.dbTableName}", as: Int32 json.field "maxUploadSize", CONFIG.size_limit - json.field "thumbnailGeneration", CONFIG.generate_thumbnails - json.field "filenameLength", CONFIG.filename_length + json.field "thumbnailGeneration", CONFIG.generateThumbnails + json.field "filenameLength", CONFIG.fileameLength end end end @@ -222,10 +222,10 @@ module Handling end def delete_file(env) - if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.db_table_name} WHERE delete_key = ?)", env.params.query["key"], as: Bool + if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.dbTableName} WHERE delete_key = ?)", env.params.query["key"], as: Bool begin fileinfo = SQL.query_all("SELECT filename, extension, thumbnail - FROM #{CONFIG.db_table_name} + FROM #{CONFIG.dbTableName} WHERE delete_key = ?", env.params.query["key"], as: {filename: String, extension: String, thumbnail: String | Nil})[0] @@ -237,7 +237,7 @@ module Handling File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") end # Delete entry from db - SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"] + SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE delete_key = ?", env.params.query["key"] LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" msg("File '#{fileinfo[:filename]}' deleted successfully") rescue ex diff --git a/src/http-errors.cr b/src/http-errors.cr index 023d3a5..9293971 100644 --- a/src/http-errors.cr +++ b/src/http-errors.cr @@ -2,39 +2,39 @@ macro error401(message) env.response.content_type = "application/json" env.response.status_code = 401 error_message = {"error" => {{message}}}.to_json - return error_message + error_message end macro error403(message) env.response.content_type = "application/json" env.response.status_code = 403 error_message = {"error" => {{message}}}.to_json - return error_message + error_message end macro error404(message) env.response.content_type = "application/json" env.response.status_code = 404 error_message = {"error" => {{message}}}.to_json - return error_message + error_message end macro error413(message) env.response.content_type = "application/json" env.response.status_code = 413 error_message = {"error" => {{message}}}.to_json - return error_message + error_message end macro error500(message) env.response.content_type = "application/json" env.response.status_code = 500 error_message = {"error" => {{message}}}.to_json - return error_message + error_message end macro msg(message) env.response.content_type = "application/json" msg = {"message" => {{message}}}.to_json - return msg + msg end diff --git a/src/jobs.cr b/src/jobs.cr index b07e4e3..15cac1f 100644 --- a/src/jobs.cr +++ b/src/jobs.cr @@ -1,14 +1,26 @@ # Pretty cool way to write background jobs! :) module Jobs def self.check_old_files - if CONFIG.delete_files_after_check_seconds <= 0 + if CONFIG.deleteFilesCheck <= 0 LOGGER.info "File deletion is disabled" return end spawn do loop do Utils.check_old_files - sleep CONFIG.delete_files_after_check_seconds + sleep CONFIG.deleteFilesCheck + end + end + end + + def self.retrieve_tor_exit_nodes + if !CONFIG.blockTorAddresses + return + end + spawn do + loop do + Utils.retrieve_tor_exit_nodes + sleep CONFIG.torExitNodesCheck end end end @@ -27,6 +39,7 @@ module Jobs def self.run check_old_files + retrieve_tor_exit_nodes kemal end end diff --git a/src/routing.cr b/src/routing.cr index 39c4d43..9704fb9 100644 --- a/src/routing.cr +++ b/src/routing.cr @@ -1,11 +1,27 @@ +require "./http-errors" + module Routing - # @@ip : String = "" + @@exit_nodes = Array(String).new + if CONFIG.blockTorAddresses + spawn do + # Wait a little for Utils.retrieve_tor_exit_nodes to execute first + # or it will load an old exit node list + # I think this can be replaced by channels which makes me able to + # receive data from fibers + sleep 5 + loop do + LOGGER.debug "Updating Tor exit nodes array" + @@exit_nodes = Utils.load_tor_exit_nodes + sleep CONFIG.torExitNodesCheck + 5 + end + end + before_post do |env| + ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first + error401 CONFIG.torMessage if ip_address.includes?(ip_address) + end + end def self.register_all - # before_get "*" do |env| - # @@ip = env.request.headers["X-Real-IP"] - # end - get "/" do |env| files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 host = env.request.headers["Host"] diff --git a/src/utils.cr b/src/utils.cr index 256ecb1..3737bf0 100644 --- a/src/utils.cr +++ b/src/utils.cr @@ -2,10 +2,10 @@ module Utils extend self def create_db - if !SQL.query_one "SELECT EXISTS (SELECT name FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.db_table_name}');", as: Bool + if !SQL.query_one "SELECT EXISTS (SELECT name FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.dbTableName}');", as: Bool LOGGER.info "Creating sqlite3 database at '#{CONFIG.db}'" begin - SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.db_table_name} + SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.dbTableName} (original_filename text, filename text, extension text, uploaded_at text, checksum text, ip text, delete_key text, thumbnail text)" rescue ex LOGGER.fatal "#{ex.message}" @@ -44,10 +44,10 @@ module Utils LOGGER.info "Deleting old files" dir = Dir.new("#{CONFIG.files}") # Delete entries from DB - SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE uploaded_at < date('now', '-#{CONFIG.delete_files_after} days');" + SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE uploaded_at < date('now', '-#{CONFIG.deleteFilesAfter} days');" # Delete files dir.each_child do |file| - if (Time.utc - File.info("#{CONFIG.files}/#{file}").modification_time).days >= CONFIG.delete_files_after + if (Time.utc - File.info("#{CONFIG.files}/#{file}").modification_time).days >= CONFIG.deleteFilesAfter LOGGER.debug "Deleting file '#{file}'" begin File.delete("#{CONFIG.files}/#{file}") @@ -64,7 +64,7 @@ module Utils def check_dependencies dependencies = ["ffmpeg"] dependencies.each do |dep| - next if !CONFIG.generate_thumbnails + next if !CONFIG.generateThumbnails if !Process.find_executable(dep) LOGGER.fatal("'#{dep}' was not found") exit(1) @@ -74,7 +74,7 @@ module Utils # TODO: # def check_duplicate(upload) - # file_checksum = SQL.query_all("SELECT checksum FROM #{CONFIG.db_table_name} WHERE original_filename = ?", upload.filename, as:String).try &.[0]? + # file_checksum = SQL.query_all("SELECT checksum FROM #{CONFIG.dbTableName} WHERE original_filename = ?", upload.filename, as:String).try &.[0]? # if file_checksum.nil? # return # else @@ -97,20 +97,20 @@ module Utils # TODO: Check if there are no other possibilities to get a random filename and exit def generate_filename - filename = Random.base58(CONFIG.filename_length) + filename = Random.base58(CONFIG.fileameLength) loop do - if SQL.query_one("SELECT COUNT(filename) FROM #{CONFIG.db_table_name} WHERE filename = ?", filename, as: Int32) == 0 + if SQL.query_one("SELECT COUNT(filename) FROM #{CONFIG.dbTableName} WHERE filename = ?", filename, as: Int32) == 0 return filename else LOGGER.debug "Filename collision! Generating a new filename" - filename = Random.base58(CONFIG.filename_length) + filename = Random.base58(CONFIG.fileameLength) end end end def generate_thumbnail(filename, extension) # Disable generation if false - return if !CONFIG.generate_thumbnails + return if !CONFIG.generateThumbnails LOGGER.debug "Generating thumbnail for #{filename + extension} in background" process = Process.run("ffmpeg", [ @@ -127,7 +127,7 @@ module Utils ]) if process.normal_exit? LOGGER.debug "Thumbnail for #{filename + extension} generated successfully" - SQL.exec "UPDATE #{CONFIG.db_table_name} SET thumbnail = ? WHERE filename = ?", filename + ".jpg", filename + SQL.exec "UPDATE #{CONFIG.dbTableName} SET thumbnail = ? WHERE filename = ?", filename + ".jpg", filename else end end @@ -146,22 +146,22 @@ module Utils end def delete_file(env) - fileinfo = SQL.query_all("SELECT filename, extension, thumbnail - FROM #{CONFIG.db_table_name} + fileinfo = SQL.query_all("SELECT filename, extension, thumbnail + FROM #{CONFIG.dbTableName} WHERE delete_key = ?", - env.params.query["key"], - as: {filename: String, extension: String, thumbnail: String | Nil})[0] + env.params.query["key"], + as: {filename: String, extension: String, thumbnail: String | Nil})[0] - # Delete file - File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}") - if fileinfo[:thumbnail] - # Delete thumbnail - File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") - end - # Delete entry from db - SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"] - LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" - msg("File '#{fileinfo[:filename]}' deleted successfully") + # Delete file + File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}") + if fileinfo[:thumbnail] + # Delete thumbnail + File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}") + end + # Delete entry from db + SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE delete_key = ?", env.params.query["key"] + LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}" + msg("File '#{fileinfo[:filename]}' deleted successfully") end def detect_extension(file) : String @@ -185,4 +185,25 @@ module Utils end "" end + + def retrieve_tor_exit_nodes + LOGGER.debug "Retrieving Tor exit nodes list" + resp = HTTP::Client.get(CONFIG.torExitNodesUrl) do |res| + if res.success? && res.status_code == 200 + begin + File.open(CONFIG.torExitNodesFile, "w") do |output| + IO.copy(res.body_io, output) + end + rescue ex + LOGGER.error "Failed to write to file: #{ex.message}" + end + else + LOGGER.error "Failed to retrieve exit nodes list. Status Code: #{res.status_code}" + end + end + end + + def load_tor_exit_nodes + exit_nodes = File.read_lines(CONFIG.torExitNodesFile) + end end diff --git a/src/views/index.ecr b/src/views/index.ecr index bd152ee..a001b8c 100644 --- a/src/views/index.ecr +++ b/src/views/index.ecr @@ -25,6 +25,9 @@ Chatterino Config | ShareX Config | file-uploader-crystal (BETA <%= CURRENT_TAG %> - <%= CURRENT_VERSION %> @ <%= CURRENT_BRANCH %>)

Archivos alojados: <%= files_hosted %>

+ <% if CONFIG.blockTorAddresses %> +

<%= CONFIG.torMessage %>

+ <% end %>