0.9.0: Mode admin endpoints, use UTC date again since sqlite doesn't support HTTP dates, use official tor exit nodes list

config/config.yml

@@ -1,6 +1,6 @@
 files: "./files"
 thumbnails: "./thumbnails"
-generateThumbnails: false
+generateThumbnails: true
 db: "./db.sqlite3"
 dbTableName: "files"
 adminEnabled: true
@@ -11,8 +11,8 @@ size_limit: 512
 port: 8080
 blockTorAddresses: true
 # Every hour
-torExitNodesCheck: 3600
+torExitNodesCheck: 1600
-torExitNodesUrl: "https://www.dan.me.uk/torlist/?exit"
+torExitNodesUrl: "https://check.torproject.org/exit-addresses"
 torExitNodesFile: "./torexitnodes.txt"
 torMessage: "TOR IS BLOCKED!"
 filesPerIP: 2
@@ -22,9 +22,9 @@ rateLimitMessage: ""
 # If you define the unix socket, it will only listen on the socket and not the port.
 #unix_socket: "/tmp/file-uploader.sock"
 # In days
-deleteFilesAfter: 7
+deleteFilesAfter: 1
 # In seconds
-deleteFilesCheck: 1800
+deleteFilesCheck: 1600
 deleteKeyLength: 4
 siteInfo: "Whatever you want to put here"
 siteWarning: "WARNING!"

src/config.cr

@@ -35,9 +35,8 @@ class Config
   property blockTorAddresses : Bool? = false
   # How often (in seconds) should this program download the exit nodes list
   property torExitNodesCheck : Int32 = 3600
-  # A URL with a list of exit nodes addresses
+  # Only https://check.torproject.org/exit-addresses is supported
-  # The list needs to contain a IP address per line
+  property torExitNodesUrl : String = "https://check.torproject.org/exit-addresses"
-  property torExitNodesUrl : String = "https://www.dan.me.uk/torlist/?exit"
   # Where the file of the exit nodes will be located, can be placed anywhere
   property torExitNodesFile : String = "./torexitnodes.txt"
   # Message that will be displayed to the Tor user.

src/handling/admin.cr

@@ -3,7 +3,12 @@ require "../http-errors"
 module Handling::Admin
   extend self
 
+  #   private macro json_fill(named_tuple, field_name)
+  #     j.field {{field_name}}, {{named_tuple}}[:{{field_name}}]
+  #   end
+
   # /api/admin/delete
+  # curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: asd" http://localhost:8080/api/admin/delete -d '{"files": ["j63"]}' | jq
   def delete_file(env)
     files = env.params.json["files"].as((Array(JSON::Any)))
     successfull_files = [] of String
@@ -46,20 +51,22 @@ module Handling::Admin
   end
 
   # /api/admin/deleteiplimit
+  # curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: asd" http://localhost:8080/api/admin/deleteiplimit -d '{"ips": ["127.0.0.1"]}' | jq
+
   def delete_ip_limit(env)
-    ips = env.params.json["ips"].as((Array(JSON::Any)))
+    data = env.params.json["ips"].as((Array(JSON::Any)))
-    successfull_ips = [] of String
+    successfull = [] of String
-    failed_ips = [] of String
+    failed = [] of String
-    ips.each do |ip|
+    data.each do |item|
-      ip = ip.to_s
+      item = item.to_s
       begin
         # Delete entry from db
-        SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", ip
+        SQL.exec "DELETE FROM #{CONFIG.ipTableName} WHERE ip = ?", item
-        LOGGER.debug "Rate limit for '#{ip}' was deleted"
+        LOGGER.debug "Rate limit for '#{item}' was deleted"
-        successfull_ips << ip
+        successfull << item
       rescue ex : DB::NoResultsError
-        LOGGER.error("Rate limit for '#{ip}' doesn't exist or is not registered in the database: #{ex.message}")
+        LOGGER.error("Rate limit for '#{item}' doesn't exist or is not registered in the database: #{ex.message}")
-        failed_ips << ip
+        failed << item
       rescue ex
         LOGGER.error "Unknown error: #{ex.message}"
         error500 "Unknown error: #{ex.message}"
@@ -67,11 +74,101 @@ module Handling::Admin
     end
     json = JSON.build do |j|
       j.object do
-        j.field "successfull", successfull_ips.size
+        j.field "successfull", successfull.size
-        j.field "failed", failed_ips.size
+        j.field "failed", failed.size
-        j.field "successfullUnbans", successfull_ips
+        j.field "successfullUnbans", successfull
-        j.field "failedUnbans", failed_ips
+        j.field "failedUnbans", failed
       end
     end
   end
+
+  # /api/admin/fileinfo
+  # curl -X POST -H "Content-Type: application/json" -H "X-Api-Key: asd" http://localhost:8080/api/admin/fileinfo -d '{"files": ["j63"]}' | jq
+  def retrieve_file_info(env)
+    data = env.params.json["files"].as((Array(JSON::Any)))
+    successfull = [] of NamedTuple(original_filename: String, filename: String, extension: String,
+      uploaded_at: String, checksum: String, ip: String, delete_key: String,
+      thumbnail: String | Nil)
+    failed = [] of String
+    data.each do |item|
+      item = item.to_s
+      begin
+        fileinfo = SQL.query_one("SELECT original_filename, filename, extension,
+	  uploaded_at, checksum, ip, delete_key, thumbnail
+	  FROM #{CONFIG.dbTableName}
+	  WHERE filename = ?",
+          item,
+          as: {original_filename: String, filename: String, extension: String,
+               uploaded_at: String, checksum: String, ip: String, delete_key: String,
+               thumbnail: String | Nil})
+        successfull << fileinfo
+      rescue ex : DB::NoResultsError
+        LOGGER.error("File '#{item}' is not registered in the database: #{ex.message}")
+        failed << item
+      rescue ex
+        LOGGER.error "Unknown error: #{ex.message}"
+        error500 "Unknown error: #{ex.message}"
+      end
+    end
+    json = JSON.build do |j|
+      j.object do
+        j.field "files" do
+          j.array do
+            successfull.each do |fileinfo|
+              j.object do
+                j.field fileinfo[:filename] do
+                  j.object do
+                    j.field "original_filename", fileinfo[:original_filename]
+                    j.field "filename", fileinfo[:filename]
+                    j.field "extension", fileinfo[:extension]
+                    j.field "uploaded_at", fileinfo[:uploaded_at]
+                    j.field "checksum", fileinfo[:checksum]
+                    j.field "ip", fileinfo[:ip]
+                    j.field "delete_key", fileinfo[:delete_key]
+                    j.field "thumbnail", fileinfo[:thumbnail]
+                  end
+                end
+              end
+            end
+          end
+        end
+        j.field "successfull", successfull.size
+        j.field "failed", failed.size
+        # j.field "successfullFiles"
+        j.field "failedFiles", failed
+      end
+    end
+  end
+
+  # /api/admin/torexitnodes
+  # curl -X GET -H "X-Api-Key: asd" http://localhost:8080/api/admin/torexitnodes | jq
+  def retrieve_tor_exit_nodes(env, nodes)
+    json = JSON.build do |j|
+      j.object do
+        j.field "ips", nodes
+      end
+    end
+  end
+
+  # /api/admin/whitelist
+  # curl -X GET -H "X-Api-Key: asd" http://localhost:8080/api/admin/torexitnodes | jq
+#   def add_ip_to_whitelist(env, nodes)
+#     json = JSON.build do |j|
+#       j.object do
+#         j.field "ips", nodes
+#       end
+#     end
+#   end
+
+  # /api/admin/blacklist
+  # curl -X GET -H "X-Api-Key: asd" http://localhost:8080/api/admin/torexitnodes | jq
+  def add_ip_to_blacklist(env, nodes)
+    json = JSON.build do |j|
+      j.object do
+        j.field "ips", nodes
+      end
+    end
+  end
+
+  # MODULE END
 end

src/handling/handling.cr

@@ -1,5 +1,6 @@
 require "../http-errors"
 require "http/client"
+require "benchmark"
 
 module Handling
   extend self
@@ -43,7 +44,7 @@ module Handling
         IO.copy(upload.body, output)
       end
       original_filename = upload.filename
-      uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
+      uploaded_at = Time.utc
       checksum = Utils.hash_file(file_path)
     end
     # X-Forwarded-For if behind a reverse proxy and the header is set in the reverse
@@ -101,7 +102,7 @@ module Handling
       original_filename = ""
       extension = ""
       checksum = ""
-      uploaded_at = Time::Format::HTTP_DATE.format(Time.utc)
+      uploaded_at = Time.utc
       extension = File.extname(URI.parse(url).path)
       delete_key = nil
       file_path = ::File.join ["#{CONFIG.files}", filename + extension]
@@ -123,7 +124,7 @@ module Handling
         File.rename(file_path, file_path + extension)
         file_path = ::File.join ["#{CONFIG.files}", filename + extension]
       end
-      # TODO: Benchmark this:
+      # The second one is faster and it uses less memory
       # original_filename = URI.parse("https://ayaya.beauty/PqC").path.split("/").last
       original_filename = url.split("/").last
       checksum = Utils.hash_file(file_path)
@@ -177,10 +178,21 @@ module Handling
       WHERE filename = ?",
         env.params.url["filename"].split(".").first,
         as: {filename: String, ofilename: String, up_at: String, ext: String, checksum: String, thumbnail: String | Nil})[0]
-
+      #   Benchmark.ips do |x|
-      headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"})
+      #     x.report("header multiple") { headers(env, {"Content-Disposition" => "inline; filename*=UTF-8''#{fileinfo[:ofilename]}",
-      headers(env, {"Last-Modified" => "#{fileinfo[:up_at]}"})
+      # 	"Last-Modified" => "#{fileinfo[:up_at]}",
-      headers(env, {"ETag" => "#{fileinfo[:checksum]}"})
+      # 	"ETag" => "#{fileinfo[:checksum]}"}) }
+      #     x.report("shorter sleep") do
+      # 		env.response.headers["Content-Disposition"] = "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"
+      # 		env.response.headers["Last-Modified"] = "#{fileinfo[:up_at]}"
+      # 		env.response.headers["ETag"] = "#{fileinfo[:checksum]}"
+      # 	end
+      #   end
+      # `env.response.headers` is faster than `headers(env, Hash(String, String))`
+      # https://github.com/kemalcr/kemal/blob/3243b8e0e03568ad3bd9f0ad6f445c871605b821/src/kemal/helpers/helpers.cr#L102C1-L104C4
+      env.response.headers["Content-Disposition"] = "inline; filename*=UTF-8''#{fileinfo[:ofilename]}"
+      #   env.response.headers["Last-Modified"] = "#{fileinfo[:up_at]}"
+      env.response.headers["ETag"] = "#{fileinfo[:checksum]}"
 
       CONFIG.opengraphUseragents.each do |useragent|
         if env.request.headers.try &.["User-Agent"].includes?(useragent)

src/jobs.cr

@@ -20,8 +20,8 @@ module Jobs
     spawn do
       loop do
         Utils.retrieve_tor_exit_nodes
-		# Updates the @@exit_nodes array instantly
+        # Updates the @@exit_nodes array instantly
-		Routing.reload_exit_nodes
+        Routing.reload_exit_nodes
         sleep CONFIG.torExitNodesCheck
       end
     end

src/routing.cr

@@ -7,6 +7,7 @@ module Routing
   def reload_exit_nodes
     LOGGER.debug "Updating Tor exit nodes array"
     @@exit_nodes = Utils.load_tor_exit_nodes
+    LOGGER.debug "IPs inside the exit nodes array: #{@@exit_nodes.size}"
   end
 
   before_post "/api/admin/*" do |env|
@@ -94,4 +95,12 @@ module Routing
   post "/api/admin/deleteiplimit" do |env|
     Handling::Admin.delete_ip_limit(env)
   end
+
+  post "/api/admin/fileinfo" do |env|
+    Handling::Admin.retrieve_file_info(env)
+  end
+
+    get "/api/admin/torexitnodes" do |env|
+    Handling::Admin.retrieve_tor_exit_nodes(env, @@exit_nodes)
+  end
 end

src/utils.cr

@@ -181,21 +181,21 @@ module Utils
     ".wmv"  => "󠀀3026b2758e66cf11",
     ".flv"  => "󠀀464c5601",
     ".mpeg" => "000001bx",
-	# Audio
+    # Audio
-    ".mp3" => "󠀀494433",
+    ".mp3"  => "󠀀494433",
-    ".aac" => "󠀀fff1",
+    ".aac"  => "󠀀fff1",
-    ".wav" => "󠀀57415645666d7420",
+    ".wav"  => "󠀀57415645666d7420",
     ".flac" => "󠀀664c614300000022",
-    ".ogg" => "󠀀4f67675300020000000000000000",
+    ".ogg"  => "󠀀4f67675300020000000000000000",
-    ".wma" => "󠀀3026b2758e66cf11a6d900aa0062ce6c",
+    ".wma"  => "󠀀3026b2758e66cf11a6d900aa0062ce6c",
     ".aiff" => "󠀀464f524d00",
     # Whatever
     ".7z"  => "377abcaf271c",
     ".gz"  => "1f8b",
     ".iso" => "󠀀4344303031",
-	# Documents
+    # Documents
-	"pdf" => "󠀀25504446",
+    "pdf"  => "󠀀25504446",
-	"html" => "<!DOCTYPE html>",
+    "html" => "<!DOCTYPE html>",
   }
 
   def detect_extension(file) : String
@@ -235,6 +235,13 @@ module Utils
 
   def load_tor_exit_nodes
     exit_nodes = File.read_lines(CONFIG.torExitNodesFile)
+    ips = [] of String
+    exit_nodes.each do |line|
+      if line.includes?("ExitAddress")
+        ips << line.split(" ")[1]
+      end
+    end
+    return ips
   end
 
   def ip_address(env) : String