0.9.3: BUGFIX! Fix deletion of thumbnails on check_old_files job.

- Add colors to logs
- Use static table names instead of config provided ones, it's kinda
  stupid to give the user an option to set the name of the table if I'm
  developing it for sqlite

.gitignore

@@ -5,3 +5,5 @@
 *.dwarf
 data
 torexitnodes.txt
+files
+thumbnails

config/config.example.yml

@@ -1,3 +1,4 @@
+colorize_logs: true
 files: "./files"
 thumbnails: "./thumbnails"
 generateThumbnails: true
@@ -15,6 +16,7 @@ torExitNodesCheck: 1600
 torExitNodesUrl: "https://check.torproject.org/exit-addresses"
 torExitNodesFile: "./torexitnodes.txt"
 torMessage: "TOR IS BLOCKED!"
+# Set this to 0 to disable rate limiting
 filesPerIP: 2
 ipTableName: "ips"
 rateLimitPeriod: 20
@@ -38,7 +40,9 @@ opengraphUseragents:
   - "chatterino-api-cache/"
   - "FFZBot/"
   - "Twitterbot/"
+  - "Synapse/"
+  - "Mastodon/"
 
+# You can leave it empty, or add your own domains.
 alternativeDomains:
-  - "ayaya.beauty"
+  - "example.com"
-  - "lamartina.gay"

src/config.cr

@@ -3,6 +3,8 @@ require "yaml"
 class Config
   include YAML::Serializable
 
+  # Colorize logs
+  property colorize_logs : Bool = true
   # Where the uploaded files will be located
   property files : String = "./files"
   # Where the thumbnails will be located when they are successfully generated
@@ -12,8 +14,6 @@ class Config
   property generateThumbnails : Bool = false
   # Where the SQLITE3 database will be located
   property db : String = "./db.sqlite3"
-  # Name of the table that will be used for file information
-  property dbTableName : String = "files"
   # Enable or disable the admin API
   property adminEnabled : Bool = false
   # The API key for admin routes. It's passed as a "X-Api-Key" header to the
@@ -45,8 +45,6 @@ class Config
   property torMessage : String? = "Tor is blocked!"
   # How many files an IP address can upload to the server
   property filesPerIP : Int32 = 32
-  # Name of the table that will be used for rate limit information
-  property ipTableName : String = "ips"
   # How often is the file limit per IP reset? (in seconds)
   property rateLimitPeriod : Int32 = 600
   # TODO: UNUSED CONSTANT

src/file-uploader-crystal.cr

@@ -18,7 +18,7 @@ Kemal.config.port = CONFIG.port
 Kemal.config.shutdown_message = false
 Kemal.config.app_name = "file-uploader-crystal"
 # https://github.com/iv-org/invidious/blob/90e94d4e6cc126a8b7a091d12d7a5556bfe369d5/src/invidious.cr#L136C1-L136C61
-LOGGER = LogHandler.new(STDOUT, CONFIG.log_level)
+LOGGER = LogHandler.new(STDOUT, CONFIG.log_level, CONFIG.colorize_logs)
 # Give me a 128 bit CPU
 # MAX_FILES = 58**CONFIG.fileameLength
 SQL = DB.open("sqlite3://#{CONFIG.db}")

src/handling/admin.cr

@@ -17,7 +17,7 @@ module Handling::Admin
       file = file.to_s
       begin
         fileinfo = SQL.query_one("SELECT filename, extension, thumbnail
-        FROM #{CONFIG.dbTableName}
+        FROM files
         WHERE filename = ?",
           file,
           as: {filename: String, extension: String, thumbnail: String | Nil})
@@ -29,7 +29,7 @@ module Handling::Admin
           File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}")
         end
         # Delete entry from db
-        SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE filename = ?", file
+        SQL.exec "DELETE FROM files WHERE filename = ?", file
         LOGGER.debug "File '#{fileinfo[:filename]}' was deleted"
         successfull_files << file
       rescue ex : DB::NoResultsError
@@ -61,7 +61,7 @@ module Handling::Admin
       item = item.to_s
       begin
         # Delete entry from db
-        SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", item
+        SQL.exec "DELETE FROM ips WHERE ip = ?", item
         LOGGER.debug "Rate limit for '#{item}' was deleted"
         successfull << item
       rescue ex : DB::NoResultsError
@@ -95,7 +95,7 @@ module Handling::Admin
       begin
         fileinfo = SQL.query_one("SELECT original_filename, filename, extension,
 	  uploaded_at, checksum, ip, delete_key, thumbnail
-	  FROM #{CONFIG.dbTableName}
+	  FROM files
 	  WHERE filename = ?",
           item,
           as: {original_filename: String, filename: String, extension: String,

src/handling/handling.cr

@@ -1,6 +1,7 @@
 require "../http-errors"
 require "http/client"
 require "benchmark"
+
 # require "../filters"
 
 module Handling
@@ -62,11 +63,11 @@ module Handling
     end
     begin
       # Insert SQL data just before returning the upload information
-      SQL.exec "INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+      SQL.exec "INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
         original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
-      SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip, date) VALUES (?, ?)", ip_address, Time.utc.to_unix
+      SQL.exec "INSERT OR IGNORE INTO ips (ip, date) VALUES (?, ?)", ip_address, Time.utc.to_unix
-      # SQL.exec "INSERT OR IGNORE INTO #{CONFIG.ipTableName} (ip) VALUES ('#{ip_address}')"
+      # SQL.exec "INSERT OR IGNORE INTO ips (ip) VALUES ('#{ip_address}')"
-      SQL.exec "UPDATE #{CONFIG.ipTableName} SET count = count + 1 WHERE ip = ('#{ip_address}')"
+      SQL.exec "UPDATE ips SET count = count + 1 WHERE ip = ('#{ip_address}')"
     rescue ex
       LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
       return error500("An error ocurred when trying to insert the data into the DB")
@@ -148,7 +149,7 @@ module Handling
       end
       begin
         # Insert SQL data just before returning the upload information
-        SQL.exec("INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+        SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
           original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil)
         successfull_files << {filename:          filename,
                               original_filename: original_filename,
@@ -231,7 +232,7 @@ module Handling
     end
     begin
       # Insert SQL data just before returning the upload information
-      SQL.exec("INSERT INTO #{CONFIG.dbTableName} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+      SQL.exec("INSERT INTO files VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
         original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil)
       successfull_files << {filename:          filename,
                             original_filename: original_filename,
@@ -269,7 +270,7 @@ module Handling
       protocol = Utils.protocol(env)
       host = Utils.host(env)
       fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail
-      FROM #{CONFIG.dbTableName}
+      FROM files
       WHERE filename = ?",
         env.params.url["filename"].split(".").first,
         as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0]
@@ -330,7 +331,7 @@ module Handling
         json.object do
           json.field "stats" do
             json.object do
-              json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.dbTableName}", as: Int32
+              json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32
               json.field "maxUploadSize", CONFIG.size_limit
               json.field "thumbnailGeneration", CONFIG.generateThumbnails
               json.field "filenameLength", CONFIG.fileameLength
@@ -347,10 +348,10 @@ module Handling
   end
 
   def delete_file(env)
-    if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.dbTableName} WHERE delete_key = ?)", env.params.query["key"], as: Bool
+    if SQL.query_one "SELECT EXISTS(SELECT 1 FROM files WHERE delete_key = ?)", env.params.query["key"], as: Bool
       begin
         fileinfo = SQL.query_all("SELECT filename, extension, thumbnail
-        FROM #{CONFIG.dbTableName}
+        FROM files
         WHERE delete_key = ?",
           env.params.query["key"],
           as: {filename: String, extension: String, thumbnail: String | Nil})[0]
@@ -362,7 +363,7 @@ module Handling
           File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}")
         end
         # Delete entry from db
-        SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE delete_key = ?", env.params.query["key"]
+        SQL.exec "DELETE FROM files WHERE delete_key = ?", env.params.query["key"]
         LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}"
         return msg("File '#{fileinfo[:filename]}' deleted successfully")
       rescue ex

src/http-errors.cr

@@ -1,44 +1,33 @@
+macro http_error(status_code, message)
+  env.response.content_type = "application/json"
+  env.response.status_code = {{status_code}}
+  error_message = {"error" => {{message}}}.to_json    
+  error_message
+end
+
 macro error400(message)
-    env.response.content_type = "application/json"
+  http_error(400, {{message}})
-    env.response.status_code = 400
+end
-    error_message = {"error" => {{message}}}.to_json
-    error_message
-  end
 
 macro error401(message)
-    env.response.content_type = "application/json"
+  http_error(401, {{message}})
-    env.response.status_code = 401
+end
-    error_message = {"error" => {{message}}}.to_json
-    error_message
-  end
 
 macro error403(message)
-    env.response.content_type = "application/json"
+  http_error(403, {{message}})
-    env.response.status_code = 403
+end
-    error_message = {"error" => {{message}}}.to_json
-    error_message
-  end
 
 macro error404(message)
-    env.response.content_type = "application/json"
+  http_error(404, {{message}})
-    env.response.status_code = 404
+end
-    error_message = {"error" => {{message}}}.to_json
-    error_message
-  end
 
 macro error413(message)
-    env.response.content_type = "application/json"
+  http_error(413, {{message}})
-    env.response.status_code = 413
+end
-    error_message = {"error" => {{message}}}.to_json
-    error_message
-  end
 
 macro error500(message)
-    env.response.content_type = "application/json"
+  http_error(500, {{message}})
-    env.response.status_code = 500
+end
-    error_message = {"error" => {{message}}}.to_json
-    error_message
-  end
 
 macro msg(message)
   env.response.content_type = "application/json"

src/jobs.cr

@@ -8,7 +8,7 @@ module Jobs
     spawn do
       loop do
         Utils.check_old_files
-        sleep CONFIG.deleteFilesCheck
+        sleep CONFIG.deleteFilesCheck.seconds
       end
     end
   end
@@ -22,7 +22,7 @@ module Jobs
         Utils.retrieve_tor_exit_nodes
         # Updates the @@exit_nodes array instantly
         Routing.reload_exit_nodes
-        sleep CONFIG.torExitNodesCheck
+        sleep CONFIG.torExitNodesCheck.seconds
       end
     end
   end

src/logger.cr

@@ -1,4 +1,6 @@
 # https://github.com/iv-org/invidious/blob/master/src/invidious/helpers/logger.cr
+require "colorize"
+
 enum LogLevel
   All   = 0
   Trace = 1
@@ -11,7 +13,9 @@ enum LogLevel
 end
 
 class LogHandler < Kemal::BaseLogHandler
-  def initialize(@io : IO = STDOUT, @level = LogLevel::Debug)
+  def initialize(@io : IO = STDOUT, @level = LogLevel::Debug, use_color : Bool = true)
+    Colorize.enabled = use_color
+    Colorize.on_tty_only!
   end
 
   def call(context : HTTP::Server::Context)
@@ -35,28 +39,27 @@ class LogHandler < Kemal::BaseLogHandler
     context
   end
 
-  def puts(message : String)
-    @io << message << '\n'
-    @io.flush
-  end
-
   def write(message : String)
     @io << message
     @io.flush
   end
 
-  def set_log_level(level : String)
+  def color(level)
-    @level = LogLevel.parse(level)
+    case level
-  end
+    when LogLevel::Trace then :cyan
-
+    when LogLevel::Debug then :green
-  def set_log_level(level : LogLevel)
+    when LogLevel::Info  then :white
-    @level = level
+    when LogLevel::Warn  then :yellow
+    when LogLevel::Error then :red
+    when LogLevel::Fatal then :magenta
+    else                      :default
+    end
   end
 
   {% for level in %w(trace debug info warn error fatal) %}
     def {{level.id}}(message : String)
       if LogLevel::{{level.id.capitalize}} >= @level
-        puts("#{Time.utc} [{{level.id}}] #{message}")
+        puts("#{Time.utc} [{{level.id}}] #{message}".colorize(color(LogLevel::{{level.id.capitalize}})))
       end
     end
   {% end %}

src/routing.cr

@@ -27,14 +27,16 @@ module Routing
     # There is a better way to do this
     if env.request.resource == "/upload"
       begin
-        ip_info = SQL.query_all("SELECT ip, count, date FROM #{CONFIG.ipTableName} WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32})[0]
+        ip_info = SQL.query_all("SELECT ip, count, date FROM ips WHERE ip = ?", Utils.ip_address(env), as: {ip: String, count: Int32, date: Int32})[0]
         time_since_first_upload = Time.utc.to_unix - ip_info[:date]
         time_until_unban = ip_info[:date] - Time.utc.to_unix + CONFIG.rateLimitPeriod
         if time_since_first_upload > CONFIG.rateLimitPeriod
-          SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", ip_info[:ip]
+          SQL.exec "DELETE FROM ips WHERE ip = ?", ip_info[:ip]
         end
-        if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod
+        if CONFIG.filesPerIP > 0
-          halt env, status_code: 401, response: error401("Rate limited! Try again in #{time_until_unban} seconds")
+          if ip_info[:count] >= CONFIG.filesPerIP && time_since_first_upload < CONFIG.rateLimitPeriod
+            halt env, status_code: 401, response: error401("Rate limited! Try again in #{time_until_unban} seconds")
+          end
         end
       rescue ex
         LOGGER.error "Error when trying to enforce rate limits: #{ex.message}"
@@ -46,14 +48,14 @@ module Routing
   def register_all
     get "/" do |env|
       host = Utils.host(env)
-      files_hosted = SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.dbTableName}", as: Int32
+      files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32
       render "src/views/index.ecr"
     end
 
     get "/chatterino" do |env|
       host = Utils.host(env)
       protocol = Utils.protocol(env)
-      files_hosted = SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.dbTableName}", as: Int32
+      files_hosted = SQL.query_one "SELECT COUNT (filename) FROM files", as: Int32
       render "src/views/chatterino.ecr"
     end
 

src/utils.cr

@@ -2,13 +2,13 @@ module Utils
   extend self
 
   def create_db
-    if !SQL.query_one "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.dbTableName}')
+    if !SQL.query_one "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='files')
-		AND EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='#{CONFIG.ipTableName}');", as: Bool
+		AND EXISTS (SELECT 1 FROM sqlite_schema WHERE type='table' AND name='ips');", as: Bool
       LOGGER.info "Creating sqlite3 database at '#{CONFIG.db}'"
       begin
-        SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.dbTableName}
+        SQL.exec "CREATE TABLE IF NOT EXISTS files
 		(original_filename text, filename text, extension text, uploaded_at text, checksum text, ip text, delete_key text, thumbnail text)"
-        SQL.exec "CREATE TABLE IF NOT EXISTS #{CONFIG.ipTableName}
+        SQL.exec "CREATE TABLE IF NOT EXISTS ips
 		(ip text UNIQUE, count integer DEFAULT 0, date integer)"
       rescue ex
         LOGGER.fatal "#{ex.message}"
@@ -45,23 +45,23 @@ module Utils
 
   def check_old_files
     LOGGER.info "Deleting old files"
-    dir = Dir.new("#{CONFIG.files}")
+    fileinfo = SQL.query_all("SELECT filename, extension, thumbnail
-    # Delete entries from DB
+    FROM files
-    SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE uploaded_at < date('now', '-#{CONFIG.deleteFilesAfter} days');"
+    WHERE uploaded_at < datetime('now', '-#{CONFIG.deleteFilesAfter} days')",
-    # Delete files
+      as: {filename: String, extension: String, thumbnail: String | Nil})
-    dir.each_child do |file|
+
-      if (Time.utc - File.info("#{CONFIG.files}/#{file}").modification_time).days >= CONFIG.deleteFilesAfter
+    fileinfo.each do |file|
-        LOGGER.debug "Deleting file '#{file}'"
+      LOGGER.debug "Deleting file '#{file[:filename]}#{file[:extension]}'"
-        begin
+      begin
-          File.delete("#{CONFIG.files}/#{file}")
+        File.delete("#{CONFIG.files}/#{file[:filename]}#{file[:extension]}")
-        rescue ex
+        if file[:thumbnail]
-          LOGGER.error "#{ex.message}"
+          File.delete("#{CONFIG.thumbnails}/#{file[:thumbnail]}")
         end
+        SQL.exec "DELETE FROM files WHERE filename = ?", file[:filename]
+      rescue ex
+        LOGGER.error "#{ex.message}"
       end
     end
-    # Close directory to prevent `Too many open files (File::Error)` error.
-    # This is because the directory class is still saved on memory for some reason.
-    dir.close
   end
 
   def check_dependencies
@@ -77,7 +77,7 @@ module Utils
 
   # TODO:
   # def check_duplicate(upload)
-  #   file_checksum = SQL.query_all("SELECT checksum FROM #{CONFIG.dbTableName} WHERE original_filename = ?", upload.filename, as:String).try &.[0]?
+  #   file_checksum = SQL.query_all("SELECT checksum FROM files WHERE original_filename = ?", upload.filename, as:String).try &.[0]?
   #   if file_checksum.nil?
   #     return
   #   else
@@ -101,8 +101,9 @@ module Utils
   # TODO: Check if there are no other possibilities to get a random filename and exit
   def generate_filename
     filename = Random.base58(CONFIG.fileameLength)
+
     loop do
-      if SQL.query_one("SELECT COUNT(filename) FROM #{CONFIG.dbTableName} WHERE filename = ?", filename, as: Int32) == 0
+      if SQL.query_one("SELECT COUNT(filename) FROM files WHERE filename = ?", filename, as: Int32) == 0
         return filename
       else
         LOGGER.debug "Filename collision! Generating a new filename"
@@ -130,8 +131,9 @@ module Utils
       ])
     if process.exit_code == 0
       LOGGER.debug "Thumbnail for #{filename + extension} generated successfully"
-      SQL.exec "UPDATE #{CONFIG.dbTableName} SET thumbnail = ? WHERE filename = ?", filename + ".jpg", filename
+      SQL.exec "UPDATE files SET thumbnail = ? WHERE filename = ?", filename + ".jpg", filename
     else
+      # TODO: Add some sort of message when the thumbnail is not generated
     end
   end
 
@@ -159,11 +161,11 @@ module Utils
     # Delete file
     File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}")
     if fileinfo[:thumbnail]
-      # Delete thumbnail
       File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}")
     end
     # Delete entry from db
-    SQL.exec "DELETE FROM #{CONFIG.dbTableName} WHERE delete_key = ?", env.params.query["key"]
+    SQL.exec "DELETE FROM files WHERE delete_key = ?", env.params.query["key"]
+
     LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}"
     msg("File '#{fileinfo[:filename]}' deleted successfully")
   end