0.8.1: Option to disable thumbnail generation, OpenGraph implementation

for clients that support it, dynamic ShareX configuration and more.

README.md

@@ -9,7 +9,7 @@ Already replaced lol.
 - Temporary file uploads like Uguu
 - File deletion link (not available in frontend for now)
 - Chatterino and ShareX support
-- Video Thumbnails for Chatterino and FrankerFaceZ
+- Video Thumbnails for Chatterino and FrankerFaceZ (Requires `ffmpeg` to be installed, can be disabled.)
 - Unix socket support if you don't want to deal with all the TCP overhead
 - Automatic protocol detection (HTTPS or HTTP)
 - Low memory usage: Between 6MB at idle and 25MB if a file is being uploaded or retrieved. It will depend of your traffic.

config/config.yml

@@ -1,4 +1,6 @@
 files: "./files"
+thumbnails: "./thumbnails"
+generate_thumbnails: false
 db: "./db.sqlite3"
 db_table_name: "files"
 filename_length: 3

public/sharex.sxcu

@@ -1,11 +0,0 @@
-{
-  "Version": "14.0.1",
-  "DestinationType": "ImageUploader, FileUploader",
-  "RequestMethod": "POST",
-  "RequestURL": "https://ayaya.beauty/upload",
-  "Body": "MultipartFormData",
-  "FileFormName": "file",
-  "URL": "{json:link}",
-  "DeletionURL": "{json:deleteLink}",
-  "ErrorMessage": "{json:error}"
-}

src/config.cr

@@ -5,8 +5,10 @@ class Config
 
   property files : String = "./files"
   property thumbnails : String = "./thumbnails"
+  property generate_thumbnails : Bool = false
   property db : String = "./db.sqlite3"
   property db_table_name : String = "files"
+  property incremental_filename_length : Bool = true
   property filename_length : Int32 = 3
   # In MiB
   property size_limit : Int16 = 512

src/file-uploader.cr

@@ -28,6 +28,7 @@ CURRENT_BRANCH  = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
 CURRENT_COMMIT  = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
 CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}
 
+Utils.check_dependencies
 Utils.create_db
 Utils.create_files_dir
 Routing.register_all
@@ -39,7 +40,6 @@ Jobs.run
   Kemal.config.env = "production" if !ENV.has_key?("KEMAL_ENV")
 {% end %}
 
-# Set permissions to 777 so NGINX can read and write to it (BROKEN)
 if !CONFIG.unix_socket.nil?
   sleep 1.second
   LOGGER.info "Changing socket permissions to 777"
@@ -47,6 +47,7 @@ if !CONFIG.unix_socket.nil?
     File.chmod("#{CONFIG.unix_socket}", File::Permissions::All)
   rescue ex
     LOGGER.fatal "#{ex.message}"
+    exit(1)
   end
 end
 

src/handling.cr

@@ -1,4 +1,5 @@
 require "./http-errors"
+require "http/client"
 
 module Handling
   extend self
@@ -22,7 +23,10 @@ module Handling
     delete_key = nil
     # TODO: Return the file that matches a checksum inside the database
     HTTP::FormData.parse(env.request) do |upload|
-      next if upload.filename.nil? || upload.filename.to_s.empty?
+      if upload.filename.nil? || upload.filename.to_s.empty?
+        LOGGER.debug "No file provided by the user"
+        error403("No file provided")
+      end
       # TODO: upload.body is emptied when is copied or read
       # Utils.check_duplicate(upload.dup)
       extension = File.extname("#{upload.filename}")
@@ -31,71 +35,137 @@ module Handling
       end
       filename = Utils.generate_filename
       file_path = ::File.join ["#{CONFIG.files}", filename + extension]
-      File.open(file_path, "w") do |file|
+      File.open(file_path, "w") do |output|
-        IO.copy(upload.body, file)
+        IO.copy(upload.body, output)
       end
       original_filename = upload.filename
       uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
       checksum = Utils.hash_file(file_path)
-      # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
-      # proxy configuration.
-      ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first
     end
-    if !filename.empty?
+    protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
-      protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
+    host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
-      host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
+    # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
-      json = JSON.build do |j|
+    # proxy configuration.
-        j.object do
+    ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first
-          j.field "link", "#{protocol}://#{host}/#{filename}"
+    json = JSON.build do |j|
-          j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
+      j.object do
-          j.field "id", filename
+        j.field "link", "#{protocol}://#{host}/#{filename}"
-          j.field "ext", extension
+        j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
-          j.field "name", original_filename
+        j.field "id", filename
-          j.field "checksum", checksum
+        j.field "ext", extension
-          if CONFIG.delete_key_length > 0
+        j.field "name", original_filename
-            delete_key = Random.base58(CONFIG.delete_key_length)
+        j.field "checksum", checksum
-            j.field "deleteKey", delete_key
+        if CONFIG.delete_key_length > 0
-            j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
+          delete_key = Random.base58(CONFIG.delete_key_length)
-          end
+          j.field "deleteKey", delete_key
+          j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
         end
       end
-      begin
-      LOGGER.debug "Generating thumbnail in background"
-      spawn { Utils.generate_thumbnail(filename, extension) }
-      rescue ex
-         LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
-      end
-      begin
-        # Insert SQL data just before returning the upload information
-        SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
-          original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
-      rescue ex
-        LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
-        error500("An error ocurred when trying to insert the data into the DB")
-      end
-
-      return json
-    else
-      LOGGER.debug "No file provided by the user"
-      error403("No file provided")
     end
+    begin
+      spawn { Utils.generate_thumbnail(filename, extension) }
+    rescue ex
+      LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
+    end
+    begin
+      # Insert SQL data just before returning the upload information
+      SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+        original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
+    rescue ex
+      LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
+      error500("An error ocurred when trying to insert the data into the DB")
+    end
+    return json
+  end
+
+  # The most unoptimized and unstable feature lol
+  def upload_url(env)
+    env.response.content_type = "application/json"
+    extension = ""
+    filename = Utils.generate_filename
+    original_filename = ""
+    uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
+    checksum = ""
+    ip_address = env.request.headers.try &.["X-Forwarded-For"]? ? env.request.headers.["X-Forwarded-For"] : env.request.remote_address.to_s.split(":").first
+    delete_key = nil
+    # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
+    # proxy configuration.
+    if !env.params.body.nil? || env.params.body["url"].empty?
+      url = env.params.body["url"]
+      extension = File.extname(URI.parse(url).path)
+      file_path = ::File.join ["#{CONFIG.files}", filename + extension]
+      File.open(file_path, "w") do |output|
+        begin
+          HTTP::Client.get(url) do |res|
+            IO.copy(res.body_io, output)
+          end
+        rescue ex
+          LOGGER.debug "Failed to download file '#{url}': #{ex.message}"
+          error403("Failed to download file '#{url}'")
+        end
+      end
+      if extension.empty?
+        extension = Utils.detect_extension(file_path)
+        File.rename(file_path, file_path + extension)
+        file_path = ::File.join ["#{CONFIG.files}", filename + extension]
+      end
+      # TODO: Benchmark this:
+      # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last
+      original_filename = url.split("/").last
+      checksum = Utils.hash_file(file_path)
+      if !filename.empty?
+        protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
+        host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
+        json = JSON.build do |j|
+          j.object do
+            j.field "link", "#{protocol}://#{host}/#{filename}"
+            j.field "linkExt", "#{protocol}://#{host}/#{filename}#{extension}"
+            j.field "id", filename
+            j.field "ext", extension
+            j.field "name", original_filename
+            j.field "checksum", checksum
+            if CONFIG.delete_key_length > 0
+              delete_key = Random.base58(CONFIG.delete_key_length)
+              j.field "deleteKey", delete_key
+              j.field "deleteLink", "#{protocol}://#{host}/delete?key=#{delete_key}"
+            end
+          end
+        end
+        begin
+          spawn { Utils.generate_thumbnail(filename, extension) }
+        rescue ex
+          LOGGER.error "An error ocurred when trying to generate a thumbnail: #{ex.message}"
+        end
+        begin
+          # Insert SQL data just before returning the upload information
+          SQL.exec "INSERT INTO #{CONFIG.db_table_name} VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+            original_filename, filename, extension, uploaded_at, checksum, ip_address, delete_key, nil
+        rescue ex
+          LOGGER.error "An error ocurred when trying to insert the data into the DB: #{ex.message}"
+          error500("An error ocurred when trying to insert the data into the DB")
+        end
+        return json
+      end
+    else
+    end
+    error403("Data malformed")
   end
 
   def retrieve_file(env)
-      protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
+    protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
-      host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
+    host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
     begin
-      fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum
+      fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum, thumbnail
       FROM #{CONFIG.db_table_name}
       WHERE filename = ?",
-      env.params.url["filename"],
+        env.params.url["filename"].split(".").first,
-        as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String})[0]
+        as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0]
 
       headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"})
       headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"})
       headers(env, {"ETag" => "#{fileinfo[:checksum]}"})
 
-      if env.request.headers.try &.["User-Agent"].includes?("chatterino-api-cache/") || env.request.headers.try &.["User-Agent"].includes?("FFZBot/")
+      if env.request.headers.try &.["User-Agent"].includes?("chatterino-api-cache/") || env.request.headers.try &.["User-Agent"].includes?("FFZBot/") || env.request.headers.try &.["User-Agent"].includes?("Twitterbot/")
         env.response.content_type = "text/html"
         return %(
 <!DOCTYPE html>
@@ -103,7 +173,10 @@ module Handling
 <head>
     <meta charset="UTF-8">
     <meta property="og:title" content="#{fileinfo[:ofilename]}">
-    <meta property="og:image" content="#{protocol}://#{host}#{CONFIG.thumbnails.split(".")[1]}/#{fileinfo[:filename]}.jpg">
+    <meta property="og:url" content="#{protocol}://#{host}/#{fileinfo[:filename]}">
+    #{if fileinfo[:thumbnail]
+        %(<meta property="og:image" content="#{protocol}://#{host}/thumbnail/#{fileinfo[:filename]}.jpg">)
+      end}
 </head>
 </html>
 )
@@ -117,16 +190,6 @@ module Handling
 
   def retrieve_thumbnail(env)
     begin
-      # fileinfo = SQL.query_all("SELECT filename, original_filename, uploaded_at, extension, checksum
-      # FROM #{CONFIG.db_table_name}
-      # WHERE filename = ?",
-      # env.params.url["filename"],
-      #   as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String})[0]
-
-      # headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"})
-      # headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"})
-      # headers(env, {"ETag" => "#{fileinfo[:checksum]}"})
-
       send_file env, "#{CONFIG.thumbnails}/#{env.params.url["thumbnail"]}"
     rescue ex
       LOGGER.debug "Thumbnail '#{env.params.url["thumbnail"]}' does not exist: #{ex.message}"
@@ -143,6 +206,8 @@ module Handling
             json.object do
               json.field "filesHosted", SQL.query_one "SELECT COUNT (filename) FROM #{CONFIG.db_table_name}", as: Int32
               json.field "maxUploadSize", CONFIG.size_limit
+              json.field "thumbnailGeneration", CONFIG.generate_thumbnails
+              json.field "filenameLength", CONFIG.filename_length
             end
           end
         end
@@ -157,13 +222,19 @@ module Handling
   def delete_file(env)
     if SQL.query_one "SELECT EXISTS(SELECT 1 FROM #{CONFIG.db_table_name} WHERE delete_key = ?)", env.params.query["key"], as: Bool
       begin
-        fileinfo = SQL.query_all("SELECT filename, extension
+        fileinfo = SQL.query_all("SELECT filename, extension, thumbnail
         FROM #{CONFIG.db_table_name}
         WHERE delete_key = ?",
-        env.params.query["key"],
+          env.params.query["key"],
-          as: {filename: String, extension: String})[0]
+          as: {filename: String, extension: String, thumbnail: String | Nil})[0]
 
+        # Delete file
         File.delete("#{CONFIG.files}/#{fileinfo[:filename]}#{fileinfo[:extension]}")
+        if fileinfo[:thumbnail]
+          # Delete thumbnail
+          File.delete("#{CONFIG.thumbnails}/#{fileinfo[:thumbnail]}")
+        end
+        # Delete entry from db
         SQL.exec "DELETE FROM #{CONFIG.db_table_name} WHERE delete_key = ?", env.params.query["key"]
         LOGGER.debug "File '#{fileinfo[:filename]}' was deleted using key '#{env.params.query["key"]}'}"
         msg("File '#{fileinfo[:filename]}' deleted successfully")
@@ -176,4 +247,22 @@ module Handling
       error401("Delete key '#{env.params.query["key"]}' does not exist. No files were deleted")
     end
   end
+
+  def sharex_config(env)
+    protocol = env.request.headers.try &.["X-Forwarded-Proto"]? ? env.request.headers["X-Forwarded-Proto"] : "http"
+    host = env.request.headers.try &.["X-Forwarded-Host"]? ? env.request.headers["X-Forwarded-Host"] : env.request.headers["Host"]
+    env.response.content_type = "application/json"
+    env.response.headers["Content-Disposition"] = "attachment; filename=\"#{host}.sxcu\""
+    return %({
+  "Version": "14.0.1",
+  "DestinationType": "ImageUploader, FileUploader",
+  "RequestMethod": "POST",
+  "RequestURL": "#{protocol}://#{host}/upload",
+  "Body": "MultipartFormData",
+  "FileFormName": "file",
+  "URL": "{json:link}",
+  "DeletionURL": "{json:deleteLink}",
+  "ErrorMessage": "{json:error}"
+})
+  end
 end

src/routing.cr

@@ -16,11 +16,15 @@ module Routing
       Handling.upload(env)
     end
 
+    post "/api/uploadurl" do |env|
+      Handling.upload_url(env)
+    end
+
     get "/:filename" do |env|
       Handling.retrieve_file(env)
     end
 
-    get "/thumbnails/:thumbnail" do |env|
+    get "/thumbnail/:thumbnail" do |env|
       Handling.retrieve_thumbnail(env)
     end
 
@@ -28,8 +32,12 @@ module Routing
       Handling.delete_file(env)
     end
 
-    get "/stats" do |env|
+    get "/api/stats" do |env|
       Handling.stats(env)
     end
+
+    get "/sharex.sxcu" do |env|
+      Handling.sharex_config(env)
+    end
   end
 end

src/utils.cr

@@ -26,6 +26,20 @@ module Utils
     end
   end
 
+  def create_thumbnails_dir
+    if !CONFIG.thumbnails
+      if !Dir.exists?("#{CONFIG.thumbnails}")
+        LOGGER.info "Creating thumbnaisl folder under '#{CONFIG.thumbnails}'"
+        begin
+          Dir.mkdir("#{CONFIG.thumbnails}")
+        rescue ex
+          LOGGER.fatal "#{ex.message}"
+          exit(1)
+        end
+      end
+    end
+  end
+
   def check_old_files
     LOGGER.info "Deleting old files"
     dir = Dir.new("#{CONFIG.files}")
@@ -39,7 +53,7 @@ module Utils
           File.delete("#{CONFIG.files}/#{file}")
         rescue ex
           LOGGER.error "#{ex.message}"
-          end
+        end
       end
     end
     # Close directory to prevent `Too many open files (File::Error)` error.
@@ -47,6 +61,17 @@ module Utils
     dir.close
   end
 
+  def check_dependencies
+    dependencies = ["ffmpeg"]
+    dependencies.each do |dep|
+      next if !CONFIG.generate_thumbnails
+      if !Process.find_executable(dep)
+        LOGGER.fatal("'#{dep}' was not found")
+        exit(1)
+      end
+    end
+  end
+
   # TODO:
   # def check_duplicate(upload)
   #   file_checksum = SQL.query_all("SELECT checksum FROM #{CONFIG.db_table_name} WHERE original_filename = ?", upload.filename, as:String).try &.[0]?
@@ -83,22 +108,28 @@ module Utils
     end
   end
 
-  # TODO: Thumbnail generation for videos. Done but error checking IS NOT DONE
   def generate_thumbnail(filename, extension)
-    Process.run("ffmpeg",
+    # Disable generation if false
+    return if !CONFIG.generate_thumbnails
+    LOGGER.debug "Generating thumbnail for #{filename + extension} in background"
+    process = Process.run("ffmpeg",
       [
-      "-hide_banner",
+        "-hide_banner",
-      "-i",
+        "-i",
-      "#{CONFIG.files}/#{filename+extension}",
+        "#{CONFIG.files}/#{filename + extension}",
-      "-movflags", "faststart",
+        "-movflags", "faststart",
-      "-f", "mjpeg",
+        "-f", "mjpeg",
-      "-q:v", "2",
+        "-q:v", "2",
-      "-vf", "scale='min(350,iw)':'min(350,ih)':force_original_aspect_ratio=decrease, thumbnail=100",
+        "-vf", "scale='min(350,iw)':'min(350,ih)':force_original_aspect_ratio=decrease, thumbnail=100",
-      "-frames:v", "1",
+        "-frames:v", "1",
-      "-update", "1",
+        "-update", "1",
-      "#{CONFIG.thumbnails}/#{filename}.jpg"
+        "#{CONFIG.thumbnails}/#{filename}.jpg",
       ])
-    SQL.exec "UPDATE #{CONFIG.db_table_name} SET thumbnail = ? WHERE filename = ?", filename+".jpg", filename
+    if process.normal_exit?
+      LOGGER.debug "Thumbnail for #{filename + extension} generated successfully"
+      SQL.exec "UPDATE #{CONFIG.db_table_name} SET thumbnail = ? WHERE filename = ?", filename + ".jpg", filename
+    else
+    end
   end
 
   # Delete socket if the server has not been previously cleaned by the server (Due to unclean exits, crashes, etc.)
@@ -113,4 +144,26 @@ module Utils
       end
     end
   end
+
+  def detect_extension(file) : String
+    magic_bytes = {
+      ".png"  => "89504e470d0a1a0a",
+      ".jpg"  => "ffd8ff",
+      ".webm" => "1a45dfa3",
+      ".mp4"  => "66747970",
+      ".gif"  => "474946383",
+      ".7z"   => "377abcaf271c",
+      ".gz"   => "1f8b",
+    }
+    file = File.open(file)
+    slice = Bytes.new(8)
+    hex = IO::Hexdump.new(file)
+    hex.read(slice)
+    magic_bytes.each do |ext, mb|
+      if slice.hexstring.includes?(mb)
+        return ext
+      end
+    end
+    ""
+  end
 end