0.8.2: Admin API, confiugurable OpenGraph user-agents.

README.md

@@ -10,6 +10,7 @@ Already replaced lol.
 - File deletion link (not available in frontend for now)
 - Chatterino and ShareX support
 - Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, can be disabled.)
+- [Small Admin API](./src/handling/admin.cr) that allows you to delete files. (Needs to be enabled in the configuration)
 - Unix socket support if you don't want to deal with all the TCP overhead
 - Automatic protocol detection (HTTPS or HTTP)
 - Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. It will depend of your traffic.

config/config.yml

@@ -3,6 +3,8 @@ thumbnails: "./thumbnails"
 generate_thumbnails: false
 db: "./db.sqlite3"
 db_table_name: "files"
+adminEnabled: false
+adminApiKey: "asd"
 filename_length: 3
 # In MiB
 size_limit: 512
@@ -20,3 +22,9 @@ log_level: "debug"
 
 blocked_extensions:
   - "exe"
+
+# List of useragents that use OpenGraph to gather file information
+opengraph_useragents:
+  - "chatterino-api-cache/"
+  - "FFZBot/"
+  - "Twitterbot/"

src/config.cr

@@ -8,6 +8,8 @@ class Config
   property generate_thumbnails : Bool = false
   property db : String = "./db.sqlite3"
   property db_table_name : String = "files"
+  property adminEnabled : Bool = false
+  property adminApiKey : String = ""
   property incremental_filename_length : Bool = true
   property filename_length : Int32 = 3
   # In MiB
@@ -20,6 +22,7 @@ class Config
   property delete_key_length : Int32 = 4
   # Blocked extensions that are not allowed to be uploaded to the server
   property blocked_extensions : Array(String) = [] of String
+  property opengraph_useragents : Array(String) = [] of String
   property siteInfo : String = "xd"
   property siteWarning : String? = ""
   property log_level : LogLevel = LogLevel::Info

src/file-uploader.cr

@@ -8,7 +8,7 @@ require "digest"
 require "./logger"
 require "./routing"
 require "./utils"
-require "./handling"
+require "./handling/**"
 require "./config"
 require "./jobs"
 require "./lib/**"
@@ -27,6 +27,7 @@ SQL = DB.open("sqlite3://#{CONFIG.db}")
 CURRENT_BRANCH  = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
 CURRENT_COMMIT  = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
 CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}
+CURRENT_TAG     = {{ "#{`git describe --long --abbrev=7 --tags | sed 's/([^-]*)-g.*/r\1/;s/-/./g'`.strip}" }}
 
 Utils.check_dependencies
 Utils.create_db

src/handling/admin.cr

@@ -0,0 +1,49 @@
+require "../http-errors"
+
+module Handling::Admin
+  extend self
+
+  def delete_file(env)
+    if env.request.headers.try &.["X-Api-Key"]? != CONFIG.adminApiKey || nil
+        error401 "Wrong API Key"
+    end
+    files = env.params.json["files"].as((Array(JSON::Any)))
+    successfull_files = [] of String
+    failed_files = [] of String
+    files.each do |file|
+      file = file.to_s
+      begin
+        fileinfo = SQL.query_one("SELECT filename, extension, thumbnail
+        FROM #{CONFIG.db_table_name}
+        WHERE filename = ?",
+          file,
+          as: {filename: String, extension: String, thumbnail: String | Nil})
+
+        # Delete file
+        File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}")
+        if fileinfo[:thumbnail]
+          # Delete thumbnail
+          File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}")
+        end
+        # Delete entry from db
+        SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE filename = ?", file
+        LOGGER.debug "File '#{fileinfo[:filename]}' was deleted"
+        successfull_files << file
+      rescue ex : DB::NoResultsError
+        LOGGER.error("File '#{file}' doesn't exist: #{ex.message}")
+        failed_files << file
+      rescue ex
+        LOGGER.error "Unknown error: #{ex.message}"
+        error500 "Unknown error: #{ex.message}"
+      end
+    end
+    json = JSON.build do |j|
+      j.object do
+        j.field "successfull", successfull_files.size
+        j.field "failed", failed_files.size
+        j.field "successfullFiles", successfull_files
+        j.field "failedFiles", failed_files
+      end
+    end
+  end
+end

src/handling/handling.cr

@@ -0,0 +1,270 @@
+require "../http-errors"
+require "http/client"
+
+module Handling
+  extend self
+
+  def upload(env)
+    env.response.content_type = "application/json"
+    # You can modify this if you want to allow files smaller than 1MiB.
+    # This is generally a good way to check the filesize but there is a better way to do it
+    # which is inspecting the file directly (If I'm not wrong).
+    if CONFIG.size_limit > 0
+      if env.request.headers["Content-Length"].to_i > 1048576*CONFIG.size_limit
+        error413("File is too big. The maximum size allowed is #{CONFIG.size_limit}MiB")
+      end
+    end
+    filename = ""
+    extension = ""
+    original_filename = ""
+    uploaded_at = ""
+    checksum = ""
+    ip_address = ""
+    delete_key = nil
+    # TODO: Return the file that matches a checksum inside the database
+    HTTP::FormData.parse(env.request) do |upload|
+      if upload.filename.nil? || upload.filename.to_s.empty?
+        LOGGER.debug "No file provided by the user"
+        error403("No file provided")
+      end
+      # TODO: upload.body is emptied when is copied or read
+      # Utils.check_duplicate(upload.dup)
+      extension = File.extname("#{upload.filename}")
+      if CONFIG.blocked_extensions.includes?(extension.split(".")[1])
+        error401("Extension '#{extension}' is not allowed")
+      end
+      filename = Utils.generate_filename
+      file_path = ::File.join ["#{CONFIG.files}", filename + extension]
+      File.open(file_path, "w") do |output|
+        IO.copy(upload.body, output)
+      end
+      original_filename = upload.filename
+      uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
+      checksum = Utils.hash_file(file_path)
+    end
+    protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
+    host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
+    # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
+    # proxy configuration.
+    ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first
+    json = JSON.build do |j|
+      j.object do
+        j.field "link", "#{protocol}://#{host}/#{filename}"
+        j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
+        j.field "id", filename
+        j.field "ext", extension
+        j.field "name", original_filename
+        j.field "checksum", checksum
+        if CONFIG.delete_key_length > 0
+          delete_key = Random.base58(CONFIG.delete_key_length)
+          j.field "deleteKey", delete_key
+          j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
+        end
+      end
+    end
+    begin
+      spawn { Utils.generate_thumbnail(filename, extension) }
+    rescue ex
+      LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
+    end
+    begin
+      # Insert SQL data just before returning the upload information
+      SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+        original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
+    rescue ex
+      LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
+      error500("An error ocurred when trying to insert the data into the DB")
+    end
+    return json
+  end
+
+  # The most unoptimized and unstable feature lol
+  def upload_url(env)
+    env.response.content_type = "application/json"
+    extension = ""
+    filename = Utils.generate_filename
+    original_filename = ""
+    uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
+    checksum = ""
+    ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first
+    delete_key = nil
+    # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
+    # proxy configuration.
+    if !env.params.body.nil? || env.params.body["url"].empty?
+      url = env.params.body["url"]
+      extension = File.extname(URI.parse(url).path)
+      file_path = ::File.join ["#{CONFIG.files}", filename + extension]
+      File.open(file_path, "w") do |output|
+        begin
+          HTTP::Client.get(url) do |res|
+            IO.copy(res.body_io, output)
+          end
+        rescue ex
+          LOGGER.debug "Failed to download file '#{url}': #{ex.message}"
+          error403("Failed to download file '#{url}'")
+        end
+      end
+      if extension.empty?
+        extension = Utils.detect_extension(file_path)
+        File.rename(file_path, file_path + extension)
+        file_path = ::File.join ["#{CONFIG.files}", filename + extension]
+      end
+      # TODO: Benchmark this:
+      # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last
+      original_filename = url.split("/").last
+      checksum = Utils.hash_file(file_path)
+      if !filename.empty?
+        protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
+        host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
+        json = JSON.build do |j|
+          j.object do
+            j.field "link", "#{protocol}://#{host}/#{filename}"
+            j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
+            j.field "id", filename
+            j.field "ext", extension
+            j.field "name", original_filename
+            j.field "checksum", checksum
+            if CONFIG.delete_key_length > 0
+              delete_key = Random.base58(CONFIG.delete_key_length)
+              j.field "deleteKey", delete_key
+              j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
+            end
+          end
+        end
+        begin
+          spawn { Utils.generate_thumbnail(filename, extension) }
+        rescue ex
+          LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
+        end
+        begin
+          # Insert SQL data just before returning the upload information
+          SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+            original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
+        rescue ex
+          LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
+          error500("An error ocurred when trying to insert the data into the DB")
+        end
+        return json
+      end
+    else
+    end
+    error403("Data malformed")
+  end
+
+  def retrieve_file(env)
+    protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
+    host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
+    begin
+      fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail
+      FROM #{CONFIG.db_table_name}
+      WHERE filename = ?",
+        env.params.url["filename"].split(".").first,
+        as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0]
+
+      headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"})
+      headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"})
+      headers(env, {"ETag" => "#{fileinfo[:checksum]}"})
+
+      CONFIG.opengraph_useragents.each do |useragent|
+        if env.request.headers.try &.["User-Agent"].includes?(useragent)
+          env.response.content_type = "text/html"
+          return %(
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta property="og:title" content="#{fileinfo[:ofilename]}">
+    <meta property="og:url" content="#{protocol}://#{host}/#{fileinfo[:filename]}">
+    #{if fileinfo[:thumbnail]
+        %(<meta property="og:image" content="#{protocol}://#{host}/thumbnail/#{fileinfo[:filename]}.jpg">)
+      end}
+</head>
+</html>
+)
+        end
+      end
+      send_file env, "#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:ext]}"
+    rescue ex
+      LOGGER.debug "File '#{env.params.url["filename"]}' does not exist: #{ex.message}"
+      error403("File '#{env.params.url["filename"]}' does not exist")
+    end
+  end
+
+  def retrieve_thumbnail(env)
+    begin
+      send_file env, "#{CONFIG.thumbnails}/#{env.params.url["thumbnail"]}"
+    rescue ex
+      LOGGER.debug "Thumbnail '#{env.params.url["thumbnail"]}' does not exist: #{ex.message}"
+      error403("Thumbnail '#{env.params.url["thumbnail"]}' does not exist")
+    end
+  end
+
+  def stats(env)
+    env.response.content_type = "application/json"
+    begin
+      json_data = JSON.build do |json|
+        json.object do
+          json.field "stats" do
+            json.object do
+              json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.db_table_name}", as: Int32
+              json.field "maxUploadSize", CONFIG.size_limit
+              json.field "thumbnailGeneration", CONFIG.generate_thumbnails
+              json.field "filenameLength", CONFIG.filename_length
+            end
+          end
+        end
+      end
+    rescue ex
+      LOGGER.error "Unknown error: #{ex.message}"
+      error500("Unknown error")
+    end
+    json_data
+  end
+
+  def delete_file(env)
+    if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.db_table_name} WHERE delete_key = ?)", env.params.query["key"], as: Bool
+      begin
+        fileinfo = SQL.query_all("SELECT filename, extension, thumbnail
+        FROM #{CONFIG.db_table_name}
+        WHERE delete_key = ?",
+          env.params.query["key"],
+          as: {filename: String, extension: String, thumbnail: String | Nil})[0]
+
+        # Delete file
+        File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}")
+        if fileinfo[:thumbnail]
+          # Delete thumbnail
+          File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}")
+        end
+        # Delete entry from db
+        SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"]
+        LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}"
+        msg("File '#{fileinfo[:filename]}' deleted successfully")
+      rescue ex
+        LOGGER.error("Unknown error: #{ex.message}")
+        error500("Unknown error")
+      end
+    else
+      LOGGER.debug "Key '#{env.params.query["key"]}' does not exist"
+      error401("Delete key '#{env.params.query["key"]}' does not exist. No files were deleted")
+    end
+  end
+
+  def sharex_config(env)
+    protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
+    host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
+    env.response.content_type = "application/json"
+    env.response.headers["Content-Disposition"] = "attachment; filename=\"#{host}.sxcu\""
+    return %({
+  "Version": "14.0.1",
+  "DestinationType": "ImageUploader, FileUploader",
+  "RequestMethod": "POST",
+  "RequestURL": "#{protocol}://#{host}/upload",
+  "Body": "MultipartFormData",
+  "FileFormName": "file",
+  "URL": "{json:link}",
+  "DeletionURL": "{json:deleteLink}",
+  "ErrorMessage": "{json:error}"
+})
+  end
+end

src/routing.cr

@@ -20,6 +20,12 @@ module Routing
       Handling.upload_url(env)
     end
 
+    if CONFIG.adminEnabled
+      post "/api/admin/delete" do |env|
+        Handling::Admin.delete_file(env)
+      end
+    end
+
     get "/:filename" do |env|
       Handling.retrieve_file(env)
     end

src/utils.cr

@@ -29,7 +29,7 @@ module Utils
   def create_thumbnails_dir
     if !CONFIG.thumbnails
       if !Dir.exists?("#{CONFIG.thumbnails}")
-        LOGGER.info "Creating thumbnaisl folder under '#{CONFIG.thumbnails}'"
+        LOGGER.info "Creating thumbnails folder under '#{CONFIG.thumbnails}'"
         begin
           Dir.mkdir("#{CONFIG.thumbnails}")
         rescue ex
@@ -145,6 +145,25 @@ module Utils
     end
   end
 
+  def delete_file(env)
+        fileinfo = SQL.query_all("SELECT filename, extension, thumbnail
+        FROM #{CONFIG.db_table_name}
+        WHERE delete_key = ?",
+          env.params.query["key"],
+          as: {filename: String, extension: String, thumbnail: String | Nil})[0]
+
+        # Delete file
+        File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}")
+        if fileinfo[:thumbnail]
+          # Delete thumbnail
+          File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}")
+        end
+        # Delete entry from db
+        SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"]
+        LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}"
+        msg("File '#{fileinfo[:filename]}' deleted successfully")
+  end
+
   def detect_extension(file) : String
     magic_bytes = {
       ".png"  => "89504e470d0a1a0a",

src/views/index.ecr

@@ -22,7 +22,7 @@
     <div>
         <div style="text-align:center;">
         <p>
-            <a href='./chatterino.png'>Chatterino Config</a> | <a href='./sharex.sxcu'>ShareX Config</a> | <a href='https://codeberg.org/Fijxu/file-uploader-crystal'>file-uploader-crystal (BETA <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %>)</a>
+            <a href='./chatterino.png'>Chatterino Config</a> | <a href='./sharex.sxcu'>ShareX Config</a> | <a href='https://codeberg.org/Fijxu/file-uploader-crystal'>file-uploader-crystal (BETA <%= CURRENT_TAG %> - <%= CURRENT_VERSION %> @ <%= CURRENT_BRANCH %>)</a>
         </p>
         <p>Archivos alojados: <%= files_hosted %></p>
         </div>