diff --git a/README.md b/README.md index 166fffe..8c532bb 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,16 @@ I'm making this to replace my current File uploader hosted on https://ayaya.beau - Temporary file file uploader like Uguu - File deletion link (not available in frontend for now) - Chatterino and ShareX support -- Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded of retrieved. I will depend of your traffic. +- Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. I will depend of your traffic. ## TODO - ~~Add file size limit~~ ADDED - Fix error when accessing `http://127.0.0.1:8080` with an empty DB. - Better frontend... -- Disable file deletion if `delete_files_after_check_seconds` or `delete_files_after` is set to `0` -- Disable delete key if `delete_key_lenght` is `0` -- Exit if `filename_lenght` is `0` -- Disable file limit if `size_limit` is `0` -- - +- ~~Disable file deletion if `delete_files_after_check_seconds` or `delete_files_after` is set to `0`~~ DONE +- ~~Disable delete key if `delete_key_length` is `0`~~ DONE (But I think there is a better way to do it) +- ~~Exit if `filename_length` is `0`~~ DONE +- ~~Disable file limit if `size_limit` is `0`~~ DONE +- ~~Prevent files from being overwritten in the event of a name collision~~ DONE +- Dockerfile and Docker image (Crystal doesn't has dependency hell like other languages so is not really necessary to do, but useful for people that want instant deploy) diff --git a/config/config.yml b/config/config.yml new file mode 100644 index 0000000..26fcf62 --- /dev/null +++ b/config/config.yml @@ -0,0 +1,22 @@ +files: "./files" +# Set to true if running behind a reverse proxy like nignx +secure: true +db: "./db.sqlite3" +db_table_name: "files" +filename_length: 3 +# In MiB +size_limit: 512 +port: 8080 +# unix socket not implemented +unix_socket: "/run/file-uploader.sock" +# In days +delete_files_after: 7 +# In seconds +delete_files_after_check_seconds: 1800 +delete_key_length: 4 +siteInfo: "Whatever you want to put here" +siteWarning: "WARNING!" +log_level: "debug" + +blocked_extensions: + - "exe" diff --git a/src/config.cr b/src/config.cr index 9c38f2f..2d1e35d 100644 --- a/src/config.cr +++ b/src/config.cr @@ -6,7 +6,8 @@ class Config property files : String = "./files" property secure : Bool = false property db : String = "./db.sqlite3" - property filename_lenght : Int8 = 3 + property db_table_name : String = "files" + property filename_length : Int8 = 3 # In MiB property size_limit : Int16 = 512 property port : UInt16 = 8080 @@ -14,16 +15,25 @@ class Config property delete_files_after : Int32 = 7 # How often should the check of old files be performed? (in seconds) property delete_files_after_check_seconds : Int32 = 1800 - property delete_key_lenght : Int8 = 4 + property delete_key_length : Int8 = 4 # Blocked extensions that are not allowed to be uploaded to the server property blocked_extensions : Array(String) = [] of String property siteInfo : String = "xd" property siteWarning : String? = "" + property log_level : LogLevel = LogLevel::Info def self.load config_file = "config/config.yml" config_yaml = File.read(config_file) config = Config.from_yaml(config_yaml) + check_config(config) config end + + def self.check_config(config : Config) + if config.filename_length <= 0 + puts "Config: filename_length cannot be #{config.filename_length}" + exit(1) + end + end end diff --git a/src/file-uploader.cr b/src/file-uploader.cr index 7fcf35c..ed78287 100644 --- a/src/file-uploader.cr +++ b/src/file-uploader.cr @@ -5,14 +5,22 @@ require "db" require "sqlite3" require "digest" +require "./logger" +require "./routing" require "./utils" require "./handling" -require "./lib/**" require "./config" +require "./jobs" +require "./lib/**" CONFIG = Config.load Kemal.config.port = CONFIG.port SQL = DB.open("sqlite3://#{CONFIG.db}") +# https://github.com/iv-org/invidious/blob/90e94d4e6cc126a8b7a091d12d7a5556bfe369d5/src/invidious.cr#L136C1-L136C61 +# OUTPUT = File.open(File::NULL, "w") +LOGGER = LogHandler.new(STDOUT, CONFIG.log_level) +# Give me a 128 bit CPU +# MAX_FILES = 58**CONFIG.filename_length # https://github.com/iv-org/invidious/blob/90e94d4e6cc126a8b7a091d12d7a5556bfe369d5/src/invidious.cr#L78 CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }} @@ -21,38 +29,9 @@ CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./ Utils.create_db Utils.create_files_dir +Routing.register_all -get "/" do |env| - files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 - host = env.request.headers["Host"] - render "src/views/index.ecr" -end - -# TODO: Error checking later -post "/upload" do |env| - Handling.upload(env) -end - -get "/:filename" do |env| - Handling.retrieve_file(env) -end - -get "/delete" do |env| - Handling.delete_file(env) -end - -get "/stats" do |env| - Handling.stats(env) -end - -CHECK_OLD_FILES = Fiber.new do - loop do - Utils.check_old_files - sleep CONFIG.delete_files_after_check_seconds - end -end - -CHECK_OLD_FILES.enqueue +Jobs.run Kemal.run {% if flag?(:release) || flag?(:production) %} diff --git a/src/handling.cr b/src/handling.cr index f285ec9..86f5180 100644 --- a/src/handling.cr +++ b/src/handling.cr @@ -45,8 +45,10 @@ end def upload(env) env.response.content_type = "application/json" # You can modify this if you want to allow files smaller than 1MiB - if env.request.headers["Content-Length"].to_i > 1048576*CONFIG.size_limit - error413("File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB") + if CONFIG.size_limit > 0 + if env.request.headers["Content-Length"].to_i > 1048576*CONFIG.size_limit + error413("File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB") + end end filename = "" extension = "" @@ -54,7 +56,10 @@ end uploaded_at = "" file_hash = "" ip_address = "" - delete_key = Random.base58(CONFIG.delete_key_lenght) + delete_key = nil + if CONFIG.delete_key_length > 0 + delete_key = Random.base58(CONFIG.delete_key_length) + end # TODO: Return the file that matches a checksum inside the database HTTP::FormData.parse(env.request) do |upload| next if upload.filename.nil? || upload.filename.to_s.empty? @@ -64,7 +69,8 @@ end end # TODO: Check if random string is already taken by some file (This will likely # never happen but it is better to design it that way) - filename = Random.base58(CONFIG.filename_lenght) + # filename = Random.base58(CONFIG.filename_length) + filename = Utils.generate_filename if !filename.is_a?(String) error403("This doesn't look like a file") else @@ -77,7 +83,7 @@ end uploaded_at = Time.utc file_hash = Utils.hash_file(file_path) ip_address = env.request.remote_address.to_s.split(":").first - SQL.exec "INSERT INTO FILES VALUES (?, ?, ?, ?, ?, ?, ?)", + SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?)", original_filename, filename, extension, uploaded_at, file_hash, ip_address, delete_key end end @@ -90,8 +96,10 @@ end j.field "ext", extension j.field "name", original_filename j.field "checksum", file_hash - j.field "deleteKey", delete_key - j.field "deleteLink", "https://#{env.request.headers["Host"]}/delete?key=#{delete_key}" + if CONFIG.delete_key_length > 0 + j.field "deleteKey", delete_key + j.field "deleteLink", "https://#{env.request.headers["Host"]}/delete?key=#{delete_key}" + end end end else @@ -101,11 +109,17 @@ end def retrieve_file(env) begin - filename = SQL.query_one "SELECT filename FROM files WHERE filename = ?", env.params.url["filename"].to_s.split(".").first, as: String - extension = SQL.query_one "SELECT extension FROM files WHERE filename = ?", filename, as: String + LOGGER.debug "#{env.request.headers["X-Real-IP"]} /#{env.params.url["filename"]}" + rescue + LOGGER.debug "NO X-Real-IP @ /#{env.params.url["filename"]}" + end + begin + filename = SQL.query_one "SELECT filename FROM #{CONFIG.db_table_name} WHERE filename = ?", env.params.url["filename"].to_s.split(".").first, as: String + extension = SQL.query_one "SELECT extension FROM #{CONFIG.db_table_name} WHERE filename = ?", filename, as: String send_file env, "#{CONFIG.files}/#{filename}#{extension}" rescue - error403("This file does not exist") + LOGGER.debug "File #{filename} does not exists" + error403("File #{filename} does not exist") end end @@ -116,30 +130,33 @@ end json.object do json.field "stats" do json.object do - json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 + json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.db_table_name}", as: Int32 json.field "maxUploadSize", CONFIG.size_limit end end end end rescue ex - error500("Unknown error: #{ex.message}") + LOGGER.error "#{ex.message}" + error500("Unknown error") end json_data end def delete_file(env) - if SQL.query_one "SELECT EXISTS(SELECT 1 FROM files WHERE delete_key = ?)", env.params.query["key"], as: Bool + if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.db_table_name} WHERE delete_key = ?)", env.params.query["key"], as: Bool begin - file_to_delete = SQL.query_one "SELECT filename FROM files WHERE delete_key = ?", env.params.query["key"], as: String - file_extension = SQL.query_one "SELECT extension FROM files WHERE delete_key = ?", env.params.query["key"], as: String + file_to_delete = SQL.query_one "SELECT filename FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"], as: String + file_extension = SQL.query_one "SELECT extension FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"], as: String File.delete("#{CONFIG.files}/#{file_to_delete}#{file_extension}") - SQL.exec "DELETE FROM files WHERE delete_key = ?", env.params.query["key"] + SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"] + LOGGER.debug "File '#{file_to_delete}' was deleted using key '#{env.params.query["key"]}'}" msg("File '#{file_to_delete}' deleted successfully") rescue ex error500("Unknown error: #{ex.message}") end else + LOGGER.debug "Key '#{env.params.query["key"]}' does not exist" error401("Huh? This delete key doesn't exist") end end diff --git a/src/jobs.cr b/src/jobs.cr new file mode 100644 index 0000000..27949c1 --- /dev/null +++ b/src/jobs.cr @@ -0,0 +1,21 @@ +# Pretty cool way to write background jobs! :) +module Jobs + def self.check_old_files + if CONFIG.delete_files_after_check_seconds <= 0 + LOGGER.info "File deletion is disabled" + return + end + fiber = Fiber.new do + loop do + Utils.check_old_files + sleep CONFIG.delete_files_after_check_seconds + end + end + return fiber + end + + def self.run + # Tries to run the .enqueue method, if is not able to I will just not execute. + check_old_files.try &.enqueue + end +end diff --git a/src/log.cr b/src/log.cr index 91a6ce2..cf78a20 100644 --- a/src/log.cr +++ b/src/log.cr @@ -1,3 +1,2 @@ module LOGGER - -end \ No newline at end of file +end diff --git a/src/logger.cr b/src/logger.cr new file mode 100644 index 0000000..dc2e5fa --- /dev/null +++ b/src/logger.cr @@ -0,0 +1,70 @@ +# https://github.com/iv-org/invidious/blob/master/src/invidious/helpers/logger.cr +enum LogLevel + All = 0 + Trace = 1 + Debug = 2 + Info = 3 + Warn = 4 + Error = 5 + Fatal = 6 + Off = 7 +end + +class LogHandler < Kemal::BaseLogHandler + def initialize(@io : IO = STDOUT, @level = LogLevel::Debug) + end + + def call(context : HTTP::Server::Context) + elapsed_time = Time.measure { call_next(context) } + elapsed_text = elapsed_text(elapsed_time) + + # Default: full path with parameters + requested_url = context.request.resource + + # Try not to log search queries passed as GET parameters during normal use + # (They will still be logged if log level is 'Debug' or 'Trace') + if @level > LogLevel::Debug && ( + requested_url.downcase.includes?("search") || requested_url.downcase.includes?("q=") + ) + # Log only the path + requested_url = context.request.path + end + + info("#{context.response.status_code} #{context.request.method} #{requested_url} #{elapsed_text}") + + context + end + + def puts(message : String) + @io << message << '\n' + @io.flush + end + + def write(message : String) + @io << message + @io.flush + end + + def set_log_level(level : String) + @level = LogLevel.parse(level) + end + + def set_log_level(level : LogLevel) + @level = level + end + + {% for level in %w(trace debug info warn error fatal) %} + def {{level.id}}(message : String) + if LogLevel::{{level.id.capitalize}} >= @level + puts("#{Time.utc} [{{level.id}}] #{message}") + end + end + {% end %} + + private def elapsed_text(elapsed) + millis = elapsed.total_milliseconds + return "#{millis.round(2)}ms" if millis >= 1 + + "#{(millis * 1000).round(2)}µs" + end +end diff --git a/src/routing.cr b/src/routing.cr new file mode 100644 index 0000000..3986d42 --- /dev/null +++ b/src/routing.cr @@ -0,0 +1,31 @@ +module Routing + # @@ip : String = "" + + def self.register_all + # before_get "*" do |env| + # @@ip = env.request.headers["X-Real-IP"] + # end + + get "/" do |env| + files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32 + host = env.request.headers["Host"] + render "src/views/index.ecr" + end + + post "/upload" do |env| + Handling.upload(env) + end + + get "/:filename" do |env| + Handling.retrieve_file(env) + end + + get "/delete" do |env| + Handling.delete_file(env) + end + + get "/stats" do |env| + Handling.stats(env) + end + end +end diff --git a/src/utils.cr b/src/utils.cr index 00e437e..370e013 100644 --- a/src/utils.cr +++ b/src/utils.cr @@ -2,41 +2,43 @@ module Utils extend self def create_db - puts "INFO: Creating sqlite3 database at '#{CONFIG.db}'" - begin - SQL.exec "CREATE TABLE IF NOT EXISTS files + if !SQL.query_one "SELECT EXISTS (SELECT name FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.db_table_name}');", as: Bool + LOGGER.info "Creating sqlite3 database at '#{CONFIG.db}'" + begin + SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.db_table_name} (original_filename text, filename text, extension text, uploaded_at text, hash text, ip text, delete_key text)" - rescue ex - puts "ERROR: #{ex.message}" - exit + rescue ex + LOGGER.fatal "#{ex.message}" + exit(1) + end end end def create_files_dir if !Dir.exists?("#{CONFIG.files}") - puts "INFO: Creatin files folder under '#{CONFIG.files}'" + LOGGER.info "Creating files folder under '#{CONFIG.files}'" begin Dir.mkdir("#{CONFIG.files}") rescue ex - puts ex.message - exit + LOGGER.fatal "#{ex.message}" + exit(1) end end end def check_old_files - puts "INFO: Deleting old files" + LOGGER.info "Deleting old files" dir = Dir.new("#{CONFIG.files}") # Delete entries from DB - SQL.exec "DELETE FROM files WHERE uploaded_at < date('now', '-#{CONFIG.delete_files_after} days');" + SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE uploaded_at < date('now', '-#{CONFIG.delete_files_after} days');" # Delete files dir.each_child do |file| if (Time.utc - File.info("#{CONFIG.files}/#{file}").modification_time).days >= CONFIG.delete_files_after - puts "INFO: Deleting file '#{file}'" + LOGGER.debug "Deleting file '#{file}'" begin File.delete("#{CONFIG.files}/#{file}") rescue ex - puts "ERROR: #{ex.message}" + LOGGER.error "#{ex.message}" end end end @@ -54,4 +56,17 @@ module Utils return io.final.hexstring end end + + # TODO: Check if there are no other possibilities to get a random filename and exit + def generate_filename + filename = Random.base58(CONFIG.filename_length) + loop do + if SQL.query_one("SELECT COUNT(filename) FROM #{CONFIG.db_table_name} WHERE filename = ?", filename, as: Int32) == 0 + return filename + else + LOGGER.debug "Filename collision! Generating a new filename" + filename = Random.base58(CONFIG.filename_length) + end + end + end end