0.6.0: Too much things. Read README.md

README.md

@@ -8,16 +8,16 @@ I'm making this to replace my current File uploader hosted on https://ayaya.beau
 - Temporary file file uploader like Uguu
 - File deletion link (not available in frontend for now)
 - Chatterino and ShareX support
-- Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded of retrieved. I will depend of your traffic.
+- Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. I will depend of your traffic.
 
 ## TODO
 
 - ~~Add file size limit~~ ADDED
 - Fix error when accessing `http://127.0.0.1:8080` with an empty DB.
 - Better frontend...
-- Disable file deletion if `delete_files_after_check_seconds` or `delete_files_after` is set to `0`
+- ~~Disable file deletion if `delete_files_after_check_seconds` or `delete_files_after` is set to `0`~~ DONE
-- Disable delete key if `delete_key_lenght` is `0`
+- ~~Disable delete key if `delete_key_length` is `0`~~ DONE (But I think there is a better way to do it)
-- Exit if `filename_lenght` is `0`
+- ~~Exit if `filename_length` is `0`~~ DONE
-- Disable file limit if `size_limit` is `0`
+- ~~Disable file limit if `size_limit` is `0`~~ DONE
-- 
+- ~~Prevent files from being overwritten in the event of a name collision~~ DONE
-
+- Dockerfile and Docker image (Crystal doesn't has dependency hell like other languages so is not really necessary to do, but useful for people that want instant deploy)

config/config.yml

@@ -0,0 +1,22 @@
+files: "./files"
+# Set to true if running behind a reverse proxy like nignx
+secure: true
+db: "./db.sqlite3"
+db_table_name: "files"
+filename_length: 3
+# In MiB
+size_limit: 512
+port: 8080
+# unix socket not implemented
+unix_socket: "/run/file-uploader.sock"
+# In days
+delete_files_after: 7
+# In seconds
+delete_files_after_check_seconds: 1800
+delete_key_length: 4
+siteInfo: "Whatever you want to put here"
+siteWarning: "WARNING!"
+log_level: "debug"
+
+blocked_extensions:
+  - "exe"

src/config.cr

@@ -6,7 +6,8 @@ class Config
   property files : String = "./files"
   property secure : Bool = false
   property db : String = "./db.sqlite3"
-  property filename_lenght : Int8 = 3
+  property db_table_name : String = "files"
+  property filename_length : Int8 = 3
   # In MiB
   property size_limit : Int16 = 512
   property port : UInt16 = 8080
@@ -14,16 +15,25 @@ class Config
   property delete_files_after : Int32 = 7
   # How often should the check of old files be performed? (in seconds)
   property delete_files_after_check_seconds : Int32 = 1800
-  property delete_key_lenght : Int8 = 4
+  property delete_key_length : Int8 = 4
   # Blocked extensions that are not allowed to be uploaded to the server
   property blocked_extensions : Array(String) = [] of String
   property siteInfo : String = "xd"
   property siteWarning : String? = ""
+  property log_level : LogLevel = LogLevel::Info
 
   def self.load
     config_file = "config/config.yml"
     config_yaml = File.read(config_file)
     config = Config.from_yaml(config_yaml)
+    check_config(config)
     config
   end
+
+  def self.check_config(config : Config)
+    if config.filename_length <= 0
+      puts "Config: filename_length cannot be #{config.filename_length}"
+      exit(1)
+    end
+  end
 end

src/file-uploader.cr

@@ -5,14 +5,22 @@ require "db"
 require "sqlite3"
 require "digest"
 
+require "./logger"
+require "./routing"
 require "./utils"
 require "./handling"
-require "./lib/**"
 require "./config"
+require "./jobs"
+require "./lib/**"
 
 CONFIG = Config.load
 Kemal.config.port = CONFIG.port
 SQL = DB.open("sqlite3://#{CONFIG.db}")
+# https://github.com/iv-org/invidious/blob/90e94d4e6cc126a8b7a091d12d7a5556bfe369d5/src/invidious.cr#L136C1-L136C61
+# OUTPUT = File.open(File::NULL, "w")
+LOGGER = LogHandler.new(STDOUT, CONFIG.log_level)
+# Give me a 128 bit CPU
+# MAX_FILES = 58**CONFIG.filename_length
 
 # https://github.com/iv-org/invidious/blob/90e94d4e6cc126a8b7a091d12d7a5556bfe369d5/src/invidious.cr#L78
 CURRENT_BRANCH  = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
@@ -21,38 +29,9 @@ CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./
 
 Utils.create_db
 Utils.create_files_dir
+Routing.register_all
 
-get "/" do |env|
+Jobs.run
-  files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32
-  host = env.request.headers["Host"]
-  render "src/views/index.ecr"
-end
-
-# TODO: Error checking later
-post "/upload" do |env|
-  Handling.upload(env)
-end
-
-get "/:filename" do |env|
-  Handling.retrieve_file(env)
-end
-
-get "/delete" do |env|
-  Handling.delete_file(env)
-end
-
-get "/stats" do |env|
-  Handling.stats(env)
-end
-
-CHECK_OLD_FILES = Fiber.new do
-  loop do
-    Utils.check_old_files
-    sleep CONFIG.delete_files_after_check_seconds
-  end
-end
-
-CHECK_OLD_FILES.enqueue
 Kemal.run
 
 {% if flag?(:release) || flag?(:production) %}

src/handling.cr

@@ -45,8 +45,10 @@ end
   def upload(env)
     env.response.content_type = "application/json"
     # You can modify this if you want to allow files smaller than 1MiB
-    if env.request.headers["Content-Length"].to_i > 1048576*CONFIG.size_limit
+    if CONFIG.size_limit > 0
-      error413("File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB")
+      if env.request.headers["Content-Length"].to_i > 1048576*CONFIG.size_limit
+        error413("File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB")
+      end
     end
     filename = ""
     extension = ""
@@ -54,7 +56,10 @@ end
     uploaded_at = ""
     file_hash = ""
     ip_address = ""
-    delete_key = Random.base58(CONFIG.delete_key_lenght)
+    delete_key = nil
+    if CONFIG.delete_key_length > 0
+      delete_key = Random.base58(CONFIG.delete_key_length)
+    end
     # TODO: Return the file that matches a checksum inside the database
     HTTP::FormData.parse(env.request) do |upload|
       next if upload.filename.nil? || upload.filename.to_s.empty?
@@ -64,7 +69,8 @@ end
       end
       # TODO: Check if random string is already taken by some file (This will likely
       # never happen but it is better to design it that way)
-      filename = Random.base58(CONFIG.filename_lenght)
+      # filename = Random.base58(CONFIG.filename_length)
+      filename = Utils.generate_filename
       if !filename.is_a?(String)
         error403("This doesn't look like a file")
       else
@@ -77,7 +83,7 @@ end
         uploaded_at = Time.utc
         file_hash = Utils.hash_file(file_path)
         ip_address = env.request.remote_address.to_s.split(":").first
-        SQL.exec "INSERT INTO FILES VALUES (?, ?, ?, ?, ?, ?, ?)",
+        SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?)",
           original_filename, filename, extension, uploaded_at, file_hash, ip_address, delete_key
       end
     end
@@ -90,8 +96,10 @@ end
           j.field "ext", extension
           j.field "name", original_filename
           j.field "checksum", file_hash
-          j.field "deleteKey", delete_key
+          if CONFIG.delete_key_length > 0
-          j.field "deleteLink", "https://#{env.request.headers["Host"]}/delete?key=#{delete_key}"
+            j.field "deleteKey", delete_key
+            j.field "deleteLink", "https://#{env.request.headers["Host"]}/delete?key=#{delete_key}"
+          end
         end
       end
     else
@@ -101,11 +109,17 @@ end
 
   def retrieve_file(env)
     begin
-      filename = SQL.query_one "SELECT filename FROM files WHERE filename = ?", env.params.url["filename"].to_s.split(".").first, as: String
+      LOGGER.debug "#{env.request.headers["X-Real-IP"]} /#{env.params.url["filename"]}"
-      extension = SQL.query_one "SELECT extension FROM files WHERE filename = ?", filename, as: String
+    rescue
+      LOGGER.debug "NO X-Real-IP @ /#{env.params.url["filename"]}"
+    end
+    begin
+      filename = SQL.query_one "SELECT filename FROM #{CONFIG.db_table_name} WHERE filename = ?", env.params.url["filename"].to_s.split(".").first, as: String
+      extension = SQL.query_one "SELECT extension FROM #{CONFIG.db_table_name} WHERE filename = ?", filename, as: String
       send_file env, "#{CONFIG.files}/#{filename}#{extension}"
     rescue
-      error403("This file does not exist")
+      LOGGER.debug "File #{filename} does not exists"
+      error403("File #{filename} does not exist")
     end
   end
 
@@ -116,30 +130,33 @@ end
         json.object do
           json.field "stats" do
             json.object do
-              json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32
+              json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.db_table_name}", as: Int32
               json.field "maxUploadSize", CONFIG.size_limit
             end
           end
         end
       end
     rescue ex
-      error500("Unknown error: #{ex.message}")
+      LOGGER.error "#{ex.message}"
+      error500("Unknown error")
     end
     json_data
   end
 
   def delete_file(env)
-    if SQL.query_one "SELECT EXISTS(SELECT 1 FROM files WHERE delete_key = ?)", env.params.query["key"], as: Bool
+    if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.db_table_name} WHERE delete_key = ?)", env.params.query["key"], as: Bool
       begin
-        file_to_delete = SQL.query_one "SELECT filename FROM files WHERE delete_key = ?", env.params.query["key"], as: String
+        file_to_delete = SQL.query_one "SELECT filename FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"], as: String
-        file_extension = SQL.query_one "SELECT extension FROM files WHERE delete_key = ?", env.params.query["key"], as: String
+        file_extension = SQL.query_one "SELECT extension FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"], as: String
         File.delete("#{CONFIG.files}/#{file_to_delete}#{file_extension}")
-        SQL.exec "DELETE FROM files WHERE delete_key = ?", env.params.query["key"]
+        SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"]
+        LOGGER.debug "File '#{file_to_delete}' was deleted using key '#{env.params.query["key"]}'}"
         msg("File '#{file_to_delete}' deleted successfully")
       rescue ex
         error500("Unknown error: #{ex.message}")
       end
     else
+      LOGGER.debug "Key '#{env.params.query["key"]}' does not exist"
       error401("Huh? This delete key doesn't exist")
     end
   end

src/jobs.cr

@@ -0,0 +1,21 @@
+# Pretty cool way to write background jobs! :)
+module Jobs
+  def self.check_old_files
+    if CONFIG.delete_files_after_check_seconds <= 0
+      LOGGER.info "File deletion is disabled"
+      return
+    end
+    fiber = Fiber.new do
+      loop do
+        Utils.check_old_files
+        sleep CONFIG.delete_files_after_check_seconds
+      end
+    end
+    return fiber
+  end
+
+  def self.run
+    # Tries to run the .enqueue method, if is not able to I will just not execute.
+    check_old_files.try &.enqueue
+  end
+end

src/log.cr

@@ -1,3 +1,2 @@
 module LOGGER
-
+end
-end

src/logger.cr

@@ -0,0 +1,70 @@
+# https://github.com/iv-org/invidious/blob/master/src/invidious/helpers/logger.cr
+enum LogLevel
+  All   = 0
+  Trace = 1
+  Debug = 2
+  Info  = 3
+  Warn  = 4
+  Error = 5
+  Fatal = 6
+  Off   = 7
+end
+
+class LogHandler < Kemal::BaseLogHandler
+  def initialize(@io : IO = STDOUT, @level = LogLevel::Debug)
+  end
+
+  def call(context : HTTP::Server::Context)
+    elapsed_time = Time.measure { call_next(context) }
+    elapsed_text = elapsed_text(elapsed_time)
+
+    # Default: full path with parameters
+    requested_url = context.request.resource
+
+    # Try not to log search queries passed as GET parameters during normal use
+    # (They will still be logged if log level is 'Debug' or 'Trace')
+    if @level > LogLevel::Debug && (
+         requested_url.downcase.includes?("search") || requested_url.downcase.includes?("q=")
+       )
+      # Log only the path
+      requested_url = context.request.path
+    end
+
+    info("#{context.response.status_code} #{context.request.method} #{requested_url} #{elapsed_text}")
+
+    context
+  end
+
+  def puts(message : String)
+    @io << message << '\n'
+    @io.flush
+  end
+
+  def write(message : String)
+    @io << message
+    @io.flush
+  end
+
+  def set_log_level(level : String)
+    @level = LogLevel.parse(level)
+  end
+
+  def set_log_level(level : LogLevel)
+    @level = level
+  end
+
+  {% for level in %w(trace debug info warn error fatal) %}
+    def {{level.id}}(message : String)
+      if LogLevel::{{level.id.capitalize}} >= @level
+        puts("#{Time.utc} [{{level.id}}] #{message}")
+      end
+    end
+  {% end %}
+
+  private def elapsed_text(elapsed)
+    millis = elapsed.total_milliseconds
+    return "#{millis.round(2)}ms" if millis >= 1
+
+    "#{(millis * 1000).round(2)}µs"
+  end
+end

src/routing.cr

@@ -0,0 +1,31 @@
+module Routing
+  #   @@ip : String = ""
+
+  def self.register_all
+    # before_get "*" do |env|
+    #   @@ip = env.request.headers["X-Real-IP"]
+    # end
+
+    get "/" do |env|
+      files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32
+      host = env.request.headers["Host"]
+      render "src/views/index.ecr"
+    end
+
+    post "/upload" do |env|
+      Handling.upload(env)
+    end
+
+    get "/:filename" do |env|
+      Handling.retrieve_file(env)
+    end
+
+    get "/delete" do |env|
+      Handling.delete_file(env)
+    end
+
+    get "/stats" do |env|
+      Handling.stats(env)
+    end
+  end
+end

src/utils.cr

@@ -2,41 +2,43 @@ module Utils
   extend self
 
   def create_db
-    puts "INFO: Creating sqlite3 database at '#{CONFIG.db}'"
+    if !SQL.query_one "SELECT EXISTS (SELECT name FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.db_table_name}');", as: Bool
-    begin
+      LOGGER.info "Creating sqlite3 database at '#{CONFIG.db}'"
-      SQL.exec "CREATE TABLE IF NOT EXISTS files
+      begin
+        SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.db_table_name}
 		(original_filename text, filename text, extension text, uploaded_at text, hash text, ip text, delete_key text)"
-    rescue ex
+      rescue ex
-      puts "ERROR: #{ex.message}"
+        LOGGER.fatal "#{ex.message}"
-      exit
+        exit(1)
+      end
     end
   end
 
   def create_files_dir
     if !Dir.exists?("#{CONFIG.files}")
-      puts "INFO: Creatin files folder under '#{CONFIG.files}'"
+      LOGGER.info "Creating files folder under '#{CONFIG.files}'"
       begin
         Dir.mkdir("#{CONFIG.files}")
       rescue ex
-        puts ex.message
+        LOGGER.fatal "#{ex.message}"
-        exit
+        exit(1)
       end
     end
   end
 
   def check_old_files
-    puts "INFO: Deleting old files"
+    LOGGER.info "Deleting old files"
     dir = Dir.new("#{CONFIG.files}")
     # Delete entries from DB
-    SQL.exec "DELETE FROM files WHERE uploaded_at < date('now', '-#{CONFIG.delete_files_after} days');"
+    SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE uploaded_at < date('now', '-#{CONFIG.delete_files_after} days');"
     # Delete files
     dir.each_child do |file|
       if (Time.utc - File.info("#{CONFIG.files}/#{file}").modification_time).days >= CONFIG.delete_files_after
-        puts "INFO: Deleting file '#{file}'"
+        LOGGER.debug "Deleting file '#{file}'"
         begin
           File.delete("#{CONFIG.files}/#{file}")
         rescue ex
-          puts "ERROR: #{ex.message}"
+          LOGGER.error "#{ex.message}"
         end
       end
     end
@@ -54,4 +56,17 @@ module Utils
       return io.final.hexstring
     end
   end
+
+  # TODO: Check if there are no other possibilities to get a random filename and exit
+  def generate_filename
+    filename = Random.base58(CONFIG.filename_length)
+    loop do
+      if SQL.query_one("SELECT COUNT(filename) FROM #{CONFIG.db_table_name} WHERE filename = ?", filename, as: Int32) == 0
+        return filename
+      else
+        LOGGER.debug "Filename collision! Generating a new filename"
+        filename = Random.base58(CONFIG.filename_length)
+      end
+    end
+  end
 end