AppSec Component integration (#43)

Integrate AppSec Component 

---------

Co-authored-by: Sebastien Blot <sebastien@crowdsec.net>

lib/crowdsec.lua

@@ -5,8 +5,12 @@ local iputils = require "plugins.crowdsec.iputils"
 local http = require "resty.http"
 local cjson = require "cjson"
 local captcha = require "plugins.crowdsec.captcha"
+local flag = require "plugins.crowdsec.flag"
 local utils = require "plugins.crowdsec.utils"
 local ban = require "plugins.crowdsec.ban"
+local url = require "plugins.crowdsec.url"
+local bit
+if _VERSION == "Lua 5.1" then bit = require "bit" else bit = require "bit32" end
 
 -- contain runtime = {}
 local runtime = {}
@@ -21,6 +25,11 @@ runtime.timer_started = false
 
 local csmod = {}
 
+local PASSTHROUGH = "passthrough"
+local DENY = "deny"
+
+local APPSEC_API_KEY_HEADER = "x-crowdsec-appsec-api-key"
+local REMEDIATION_API_KEY_HEADER = 'x-api-key'
 
 
 -- init function
@@ -66,6 +75,30 @@ function csmod.init(configFile, userAgent)
     table.insert(runtime.conf["EXCLUDE_LOCATION"], runtime.conf["REDIRECT_LOCATION"])
   end
 
+  if runtime.conf["SSL_VERIFY"] == "false" then
+    runtime.conf["SSL_VERIFY"] = false
+  else
+    runtime.conf["SSL_VERIFY"] = true
+  end
+
+  if runtime.conf["ALWAYS_SEND_TO_APPSEC"] == "false" then
+    runtime.conf["ALWAYS_SEND_TO_APPSEC"] = false
+  else
+    runtime.conf["ALWAYS_SEND_TO_APPSEC"] = true
+  end
+
+  runtime.conf["APPSEC_ENABLED"] = false
+
+  if runtime.conf["APPSEC_URL"] ~= "" then
+    u = url.parse(runtime.conf["APPSEC_URL"])
+    runtime.conf["APPSEC_ENABLED"] = true
+    runtime.conf["APPSEC_HOST"] = u.host
+    if u.port ~= nil then
+      runtime.conf["APPSEC_HOST"] = runtime.conf["APPSEC_HOST"] .. ":" .. u.port
+    end
+    ngx.log(ngx.ERR, "APPSEC is enabled on '" .. runtime.conf["APPSEC_HOST"] .. "'")
+  end
+
 
   -- if stream mode, add callback to stream_query and start timer
   if runtime.conf["MODE"] == "stream" then
@@ -85,6 +118,8 @@ function csmod.init(configFile, userAgent)
     end
   end
 
+
+
   return true, nil
 end
 
@@ -94,16 +129,17 @@ function csmod.validateCaptcha(captcha_res, remote_ip)
 end
 
 
-local function get_http_request(link)
+local function get_remediation_http_request(link)
   local httpc = http.new()
   httpc:set_timeout(runtime.conf['REQUEST_TIMEOUT'])
   local res, err = httpc:request_uri(link, {
     method = "GET",
     headers = {
       ['Connection'] = 'close',
-      ['X-Api-Key'] = runtime.conf["API_KEY"],
+      [REMEDIATION_API_KEY_HEADER] = runtime.conf["API_KEY"],
       ['User-Agent'] = runtime.userAgent
     },
+    ssl_verify = runtime.conf["SSL_VERIFY"]
   })
   httpc:close()
   return res, err
@@ -226,7 +262,7 @@ local function stream_query(premature)
   local is_startup = runtime.cache:get("startup")
   ngx.log(ngx.DEBUG, "Stream Query from worker : " .. tostring(ngx.worker.id()) .. " with startup "..tostring(is_startup) .. " | premature: " .. tostring(premature))
   local link = runtime.conf["API_URL"] .. "/v1/decisions/stream?startup=" .. tostring(is_startup)
-  local res, err = get_http_request(link)
+  local res, err = get_remediation_http_request(link)
   if not res then
     local ok, err = ngx.timer.at(runtime.conf["UPDATE_FREQUENCY"], stream_query)
     if not ok then
@@ -321,7 +357,7 @@ end
 
 local function live_query(ip)
   local link = runtime.conf["API_URL"] .. "/v1/decisions?ip=" .. ip
-  local res, err = get_http_request(link)
+  local res, err = get_remediation_http_request(link)
   if not res then
     return true, nil, "request failed: ".. err
   end
@@ -365,6 +401,22 @@ local function live_query(ip)
   end
 end
 
+local function get_body()
+
+  ngx.req.read_body()
+  local body = ngx.req.get_body_data()
+  if body == nil then
+    local bodyfile = ngx.req.get_body_file()
+    if bodyfile then
+      local fh, err = io.open(bodyfile, "r")
+      if fh then
+        body = fh:read("*a")
+        fh:close()
+      end
+    end
+  end
+  return body
+end
 
 function csmod.GetCaptchaTemplate()
   return captcha.GetTemplate()
@@ -438,12 +490,83 @@ function csmod.allowIp(ip)
   return true, nil, nil
 end
 
-function csmod.Allow(ip)
 
+function csmod.AppSecCheck()
+  local httpc = http.new()
+  httpc:set_timeouts(runtime.conf["APPSEC_CONNECT_TIMEOUT"], runtime.conf["APPSEC_SEND_TIMEOUT"], runtime.conf["APPSEC_PROCESS_TIMEOUT"])
+
+  local uri = ngx.var.request_uri
+  local headers = ngx.req.get_headers()
+
+  -- overwrite headers with crowdsec appsec require headers
+  headers["x-crowdsec-appsec-ip"] = ngx.var.remote_addr
+  headers["x-crowdsec-appsec-host"] = ngx.var.http_host
+  headers["x-crowdsec-appsec-verb"] = ngx.var.request_method
+  headers["x-crowdsec-appsec-uri"] = uri
+  headers[APPSEC_API_KEY_HEADER] = runtime.conf["API_KEY"]
+
+  -- set CrowdSec APPSEC Host
+  headers["host"] = runtime.conf["APPSEC_HOST"]
+
+  local ok, remediation = true, "allow"
+  if runtime.conf["APPSEC_FAILURE_ACTION"] == DENY then
+    ok = false
+    remediation = runtime.conf["FALLBACK_REMEDIATION"]
+  end
+
+  local method = "GET"
+
+  local body = get_body()
+  if body ~= nil then
+    if #body > 0 then
+      method = "POST"
+      if headers["content-length"] == nil then
+        headers["content-length"] = tostring(#body)
+      end
+    end
+  end
+
+  local res, err = httpc:request_uri(runtime.conf["APPSEC_URL"], {
+    method = method,
+    headers = headers,
+    body = body,
+    ssl_verify = runtime.conf["SSL_VERIFY"],
+  })
+  httpc:close()
+
+  if err ~= nil then
+    ngx.log(ngx.ERR, "Fallback because of err: " .. err)
+    return ok, remediation, err
+  end
+
+  if res.status == 200 then
+    ok = true
+    remediation = "allow"
+  elseif res.status == 403 then
+    ok = false
+    local response = cjson.decode(res.body)
+    remediation = response.action
+  elseif res.status == 401 then
+    ngx.log(ngx.ERR, "Unauthenticated request to APPSEC")
+  else
+    ngx.log(ngx.ERR, "Bad request to APPSEC (" .. res.status .. "): " .. res.body)
+  end
+
+  return ok, remediation, err
+
+end
+
+function csmod.Allow(ip)
   if runtime.conf["ENABLED"] == "false" then
     return "Disabled", nil
   end
 
+  if ngx.req.is_internal() then
+    return
+  end
+
+  local remediationSource = flag.BOUNCER_SOURCE
+
   if utils.table_len(runtime.conf["EXCLUDE_LOCATION"]) > 0 then
     for k, v in pairs(runtime.conf["EXCLUDE_LOCATION"]) do
       if ngx.var.uri == v then
@@ -470,6 +593,23 @@ function csmod.Allow(ip)
     ngx.shared.crowdsec_cache:delete("captcha_" .. ip)
   end
 
+  -- check with appSec if the remediation component doesn't have decisions for the IP
+  -- OR
+  -- that user configured the remediation component to always check on the appSec (even if there is a decision for the IP)
+  if ok == true or runtime.conf["ALWAYS_SEND_TO_APPSEC"] == true then
+    if runtime.conf["APPSEC_ENABLED"] == true and ngx.var.no_appsec ~= "1" then
+      local appsecOk, appsecRemediation, err = csmod.AppSecCheck()
+      if err ~= nil then
+        ngx.log(ngx.ERR, "AppSec check: " .. err)
+      end
+      if appsecOk == false then
+        ok = false
+        remediationSource = flag.APPSEC_SOURCE
+        remediation = appsecRemediation
+      end
+    end
+  end
+
   local captcha_ok = runtime.cache:get("captcha_ok")
 
   if runtime.fallback ~= "" then
@@ -486,8 +626,9 @@ function csmod.Allow(ip)
 
   if captcha_ok then -- if captcha can be use (configuration is valid)
     -- we check if the IP need to validate its captcha before checking it against crowdsec local API
-    local previous_uri, state_id = ngx.shared.crowdsec_cache:get("captcha_"..ngx.var.remote_addr)
+    local previous_uri, flags = ngx.shared.crowdsec_cache:get("captcha_"..ngx.var.remote_addr)
-    if previous_uri ~= nil and state_id == captcha.GetStateID(captcha._VERIFY_STATE) then
+    local source, state_id, err = flag.GetFlags(flags)
+    if previous_uri ~= nil and state_id == flag.VERIFY_STATE then
         ngx.req.read_body()
         local captcha_res = ngx.req.get_post_args()[csmod.GetCaptchaBackendKey()] or 0
         if captcha_res ~= 0 then
@@ -496,15 +637,22 @@ function csmod.Allow(ip)
               ngx.log(ngx.ERR, "Error while validating captcha: " .. err)
             end
             if valid == true then
-                -- captcha is valid, we redirect the IP to its previous URI but in GET method
+                -- if the captcha is valid and has been applied by the application security component
-                local succ, err, forcible = ngx.shared.crowdsec_cache:set("captcha_"..ngx.var.remote_addr, previous_uri, runtime.conf["CAPTCHA_EXPIRATION"], captcha.GetStateID(captcha._VALIDATED_STATE))
+                -- then we delete the state from the cache because from the bouncing part, if the user solve the captcha
+                -- we will not propose a captcha until the 'CAPTCHA_EXPIRATION'.
+                -- But for the Application security component, we serve the captcha each time the user trigger it.
+                if source == flag.APPSEC_SOURCE then
+                  ngx.shared.crowdsec_cache:delete("captcha_"..ngx.var.remote_addr)
+                else
+                  local succ, err, forcible = ngx.shared.crowdsec_cache:set("captcha_"..ngx.var.remote_addr, previous_uri, runtime.conf["CAPTCHA_EXPIRATION"], bit.bor(flag.VALIDATED_STATE, source) )
                   if not succ then
                     ngx.log(ngx.ERR, "failed to add key about captcha for ip '" .. ngx.var.remote_addr .. "' in cache: "..err)
                   end
                   if forcible then
                     ngx.log(ngx.ERR, "Lua shared dict (crowdsec cache) is full, please increase dict size in config")
                   end
-                
+                end
+                -- captcha is valid, we redirect the IP to its previous URI but in GET method
                 ngx.req.set_method(ngx.HTTP_GET)
                 ngx.redirect(previous_uri)
                 return
@@ -514,18 +662,18 @@ function csmod.Allow(ip)
         end
     end
   end
-
   if not ok then
       if remediation == "ban" then
-        ngx.log(ngx.ALERT, "[Crowdsec] denied '" .. ngx.var.remote_addr .. "' with '"..remediation.."'")
+        ngx.log(ngx.ALERT, "[Crowdsec] denied '" .. ngx.var.remote_addr .. "' with '"..remediation.."' (by " .. flag.Flags[remediationSource] .. ")")
         ban.apply()
         return
       end
       -- if the remediation is a captcha and captcha is well configured
       if remediation == "captcha" and captcha_ok and ngx.var.uri ~= "/favicon.ico" then
-          local previous_uri, state_id = ngx.shared.crowdsec_cache:get("captcha_"..ngx.var.remote_addr)
+          local previous_uri, flags = ngx.shared.crowdsec_cache:get("captcha_"..ngx.var.remote_addr)
+          source, state_id, err = flag.GetFlags(flags)
           -- we check if the IP is already in cache for captcha and not yet validated
-          if previous_uri == nil or state_id ~= captcha.GetStateID(captcha._VALIDATED_STATE) then
+          if previous_uri == nil or remediationSource == flag.APPSEC_SOURCE then
               ngx.header.content_type = "text/html"
               ngx.header.cache_control = "no-cache"
               ngx.say(csmod.GetCaptchaTemplate())
@@ -539,7 +687,7 @@ function csmod.Allow(ip)
                   end
                 end
               end
-              local succ, err, forcible = ngx.shared.crowdsec_cache:set("captcha_"..ngx.var.remote_addr, uri , 60, captcha.GetStateID(captcha._VERIFY_STATE))
+              local succ, err, forcible = ngx.shared.crowdsec_cache:set("captcha_"..ngx.var.remote_addr, uri , 60, bit.bor(flag.VERIFY_STATE, remediationSource))
               if not succ then
                 ngx.log(ngx.ERR, "failed to add key about captcha for ip '" .. ngx.var.remote_addr .. "' in cache: "..err)
               end

lib/plugins/crowdsec/captcha.lua

@@ -3,7 +3,6 @@ local cjson = require "cjson"
 local template = require "plugins.crowdsec.template"
 local utils = require "plugins.crowdsec.utils"
 
-
 local M = {_TYPE='module', _NAME='recaptcha.funcs', _VERSION='1.0-0'}
 
 local captcha_backend_url = {}
@@ -21,30 +20,10 @@ captcha_frontend_key["recaptcha"] = "g-recaptcha"
 captcha_frontend_key["hcaptcha"] = "h-captcha"
 captcha_frontend_key["turnstile"] = "cf-turnstile"
 
-M._VERIFY_STATE = "to_verify"
-M._VALIDATED_STATE = "validated"
-
-
-M.State = {}
-M.State["1"] = M._VERIFY_STATE
-M.State["2"] = M._VALIDATED_STATE
-
 M.SecretKey = ""
 M.SiteKey = ""
 M.Template = ""
 
-
-function M.GetStateID(state)
-    for k, v in pairs(M.State) do
-        if v == state then
-            return tonumber(k)
-        end
-    end
-    return nil
-end
-
-
-
 function M.New(siteKey, secretKey, TemplateFilePath, captcha_provider)
 
     if siteKey == nil or siteKey == "" then

lib/plugins/crowdsec/config.lua

@@ -39,8 +39,8 @@ function config.loadConfig(file)
         return nil, "File ".. file .." doesn't exist"
     end
     local conf = {}
-    local valid_params = {'ENABLED','API_URL', 'API_KEY', 'BOUNCING_ON_TYPE', 'MODE', 'SECRET_KEY', 'SITE_KEY', 'BAN_TEMPLATE_PATH' ,'CAPTCHA_TEMPLATE_PATH', 'REDIRECT_LOCATION', 'RET_CODE', 'EXCLUDE_LOCATION', 'FALLBACK_REMEDIATION', 'CAPTCHA_PROVIDER'}
+    local valid_params = {'ENABLED','API_URL', 'API_KEY', 'BOUNCING_ON_TYPE', 'MODE', 'SECRET_KEY', 'SITE_KEY', 'BAN_TEMPLATE_PATH' ,'CAPTCHA_TEMPLATE_PATH', 'REDIRECT_LOCATION', 'RET_CODE', 'EXCLUDE_LOCATION', 'FALLBACK_REMEDIATION', 'CAPTCHA_PROVIDER', 'APPSEC_URL', 'APPSEC_FAILURE_ACTION', 'ALWAYS_SEND_TO_APPSEC', 'SSL_VERIFY'}
-    local valid_int_params = {'CACHE_EXPIRATION', 'CACHE_SIZE', 'REQUEST_TIMEOUT', 'UPDATE_FREQUENCY', 'CAPTCHA_EXPIRATION'}
+    local valid_int_params = {'CACHE_EXPIRATION', 'CACHE_SIZE', 'REQUEST_TIMEOUT', 'UPDATE_FREQUENCY', 'CAPTCHA_EXPIRATION', 'APPSEC_CONNECT_TIMEOUT', 'APPSEC_SEND_TIMEOUT', 'APPSEC_PROCESS_TIMEOUT'}
     local valid_bouncing_on_type_values = {'ban', 'captcha', 'all'}
     local valid_truefalse_values = {'false', 'true'}
     local default_values = {
@@ -53,7 +53,15 @@ function config.loadConfig(file)
         ['REDIRECT_LOCATION'] = "",
         ['EXCLUDE_LOCATION'] = {},
         ['RET_CODE'] = 0,
-	    ['CAPTCHA_PROVIDER'] = "recaptcha"
+        ['CAPTCHA_PROVIDER'] = "recaptcha",
+        ['APPSEC_URL'] = "",
+        ['APPSEC_CONNECT_TIMEOUT'] = 100,
+        ['APPSEC_SEND_TIMEOUT'] = 100,
+        ['APPSEC_PROCESS_TIMEOUT'] = 500,
+        ['APPSEC_FAILURE_ACTION'] = "passthrough",
+        ['SSL_VERIFY'] = "true",
+        ['ALWAYS_SEND_TO_APPSEC'] = "false",
+
     }
     for line in io.lines(file) do
         local isOk = false
@@ -75,17 +83,22 @@ function config.loadConfig(file)
             if key == "ENABLED" then
                 if not has_value(valid_truefalse_values, value) then
                     ngx.log(ngx.ERR, "unsupported value '" .. value .. "' for variable '" .. key .. "'. Using default value 'true' instead")
-			conf[key] = "true"
+                    value = "true"
                 end
             elseif key == "BOUNCING_ON_TYPE" then
                 if not has_value(valid_bouncing_on_type_values, value) then
                     ngx.log(ngx.ERR, "unsupported value '" .. value .. "' for variable '" .. key .. "'. Using default value 'ban' instead")
-			conf[key] = "ban"
+                    value = "ban"
+                end
+            elseif key == "SSL_VERIFY" then
+                if not has_value(valid_truefalse_values, value) then
+                    ngx.log(ngx.ERR, "unsupported value '" .. value .. "' for variable '" .. key .. "'. Using default value 'true' instead")
+                    value = "true"
                 end
             elseif key == "MODE" then
                 if not has_value({'stream', 'live'}, value) then
                 ngx.log(ngx.ERR, "unsupported value '" .. value .. "' for variable '" .. key .. "'. Using default value 'stream' instead")
-			conf[key] = "stream"
+                value = "stream"
                 end
             elseif key == "EXCLUDE_LOCATION" then
                 exclude_location = {}
@@ -94,15 +107,16 @@ function config.loadConfig(file)
                     table.insert(exclude_location, match)
                 end
                 end
-		    conf[key] = exclude_location
+                value = exclude_location
             elseif key == "FALLBACK_REMEDIATION" then
                 if not has_value({'captcha', 'ban'}, value) then
                 ngx.log(ngx.ERR, "unsupported value '" .. value .. "' for variable '" .. key .. "'. Using default value 'ban' instead")
-			conf[key] = "ban"
+                value = "ban"
                 end
-		else
+            end
+            
             conf[key] = value
-		end
+
         elseif has_value(valid_int_params, key) then
             conf[key] = tonumber(value)
         else

lib/plugins/crowdsec/flag.lua

@@ -0,0 +1,52 @@
+local bit
+if _VERSION == "Lua 5.1" then bit = require "bit" else bit = require "bit32" end
+
+local M = {_TYPE='module', _NAME='flag.funcs', _VERSION='1.0-0'}
+
+M.BOUNCER_SOURCE = 0x1
+M.APPSEC_SOURCE = 0x2
+M.VERIFY_STATE = 0x4
+M.VALIDATED_STATE = 0x8
+
+M.Flags = {}
+M.Flags[0x0] = ""
+M.Flags[0x1] = "bouncer"
+M.Flags[0x2] = "appsec"
+M.Flags[0x4] = "to_verify"
+M.Flags[0x8] = "validated"
+
+
+function M.GetFlags(flags)
+  local source = 0x0
+  local err = ""
+  local state = 0x0
+
+  if flags == nil then
+    return source, state, err
+  end
+
+  if bit.band(flags, M.BOUNCER_SOURCE) then
+    source = M.BOUNCER_SOURCE
+  elseif bit.band(flags, M.APPSEC_SOURCE) then
+    source = M.APPSEC_SOURCE
+  end
+
+  if bit.band(flags, M.VERIFY_STATE) then
+    state = M.VERIFY_STATE
+  elseif bit.band(flags, M.VALIDATED_STATE) then
+    state = M.VALIDATED_STATE
+  end
+  return source, state, err    
+
+end
+
+function M.GetStateID(state)
+    for k, v in pairs(M.State) do
+        if v == state then
+            return tonumber(k)
+        end
+    end
+    return nil
+end
+
+return M

lib/plugins/crowdsec/url.lua

@@ -0,0 +1,523 @@
+-- net/url.lua - a robust url parser and builder
+--
+-- Bertrand Mansion, 2011-2021; License MIT
+-- @module net.url
+-- @alias	M
+
+local M = {}
+M.version = "1.1.0"
+
+--- url options
+-- - `separator` is set to `&` by default but could be anything like `&` or `;`
+-- - `cumulative_parameters` is false by default. If true, query parameters with the same name will be stored in a table.
+-- - `legal_in_path` is a table of characters that will not be url encoded in path components
+-- - `legal_in_query` is a table of characters that will not be url encoded in query values. Query parameters only support a small set of legal characters (-_.).
+-- - `query_plus_is_space` is true by default, so a plus sign in a query value will be converted to %20 (space), not %2B (plus)
+-- @todo Add option to limit the size of the argument table
+-- @todo Add option to limit the depth of the argument table
+-- @todo Add option to process dots in parameter names, ie. `param.filter=1`
+M.options = {
+	separator = '&',
+	cumulative_parameters = false,
+	legal_in_path = {
+		[":"] = true, ["-"] = true, ["_"] = true, ["."] = true,
+		["!"] = true, ["~"] = true, ["*"] = true, ["'"] = true,
+		["("] = true, [")"] = true, ["@"] = true, ["&"] = true,
+		["="] = true, ["$"] = true, [","] = true,
+		[";"] = true
+	},
+	legal_in_query = {
+		[":"] = true, ["-"] = true, ["_"] = true, ["."] = true,
+		[","] = true, ["!"] = true, ["~"] = true, ["*"] = true,
+		["'"] = true, [";"] = true, ["("] = true, [")"] = true,
+		["@"] = true, ["$"] = true,
+	},
+	query_plus_is_space = true
+}
+
+--- list of known and common scheme ports
+-- as documented in <a href="http://www.iana.org/assignments/uri-schemes.html">IANA URI scheme list</a>
+M.services = {
+	acap     = 674,
+	cap      = 1026,
+	dict     = 2628,
+	ftp      = 21,
+	gopher   = 70,
+	http     = 80,
+	https    = 443,
+	iax      = 4569,
+	icap     = 1344,
+	imap     = 143,
+	ipp      = 631,
+	ldap     = 389,
+	mtqp     = 1038,
+	mupdate  = 3905,
+	news     = 2009,
+	nfs      = 2049,
+	nntp     = 119,
+	rtsp     = 554,
+	sip      = 5060,
+	snmp     = 161,
+	telnet   = 23,
+	tftp     = 69,
+	vemmi    = 575,
+	afs      = 1483,
+	jms      = 5673,
+	rsync    = 873,
+	prospero = 191,
+	videotex = 516
+}
+
+local function decode(str)
+	return (str:gsub("%%(%x%x)", function(c)
+		return string.char(tonumber(c, 16))
+	end))
+end
+
+local function encode(str, legal)
+	return (str:gsub("([^%w])", function(v)
+		if legal[v] then
+			return v
+		end
+		return string.upper(string.format("%%%02x", string.byte(v)))
+	end))
+end
+
+-- for query values, + can mean space if configured as such
+local function decodeValue(str)
+	if M.options.query_plus_is_space then
+		str = str:gsub('+', ' ')
+	end
+	return decode(str)
+end
+
+local function concat(a, b)
+	if type(a) == 'table' then
+		return a:build() .. b
+	else
+		return a .. b:build()
+	end
+end
+
+function M:addSegment(path)
+	if type(path) == 'string' then
+		self.path = self.path .. '/' .. encode(path:gsub("^/+", ""), M.options.legal_in_path)
+	end
+	return self
+end
+
+--- builds the url
+-- @return a string representing the built url
+function M:build()
+	local url = ''
+	if self.path then
+		local path = self.path
+		url = url .. tostring(path)
+	end
+	if self.query then
+		local qstring = tostring(self.query)
+		if qstring ~= "" then
+			url = url .. '?' .. qstring
+		end
+	end
+	if self.host then
+		local authority = self.host
+		if self.port and self.scheme and M.services[self.scheme] ~= self.port then
+			authority = authority .. ':' .. self.port
+		end
+		local userinfo
+		if self.user and self.user ~= "" then
+			userinfo = self.user
+			if self.password then
+				userinfo = userinfo .. ':' .. self.password
+			end
+		end
+		if userinfo and userinfo ~= "" then
+			authority = userinfo .. '@' .. authority
+		end
+		if authority then
+			if url ~= "" then
+				url = '//' .. authority .. '/' .. url:gsub('^/+', '')
+			else
+				url = '//' .. authority
+			end
+		end
+	end
+	if self.scheme then
+		url = self.scheme .. ':' .. url
+	end
+	if self.fragment then
+		url = url .. '#' .. self.fragment
+	end
+	return url
+end
+
+--- builds the querystring
+-- @param tab The key/value parameters
+-- @param sep The separator to use (optional)
+-- @param key The parent key if the value is multi-dimensional (optional)
+-- @return a string representing the built querystring
+function M.buildQuery(tab, sep, key)
+	local query = {}
+	if not sep then
+		sep = M.options.separator or '&'
+	end
+	local keys = {}
+	for k in pairs(tab) do
+		keys[#keys+1] = k
+	end
+	table.sort(keys, function (a, b)
+  		local function padnum(n, rest) return ("%03d"..rest):format(tonumber(n)) end
+  		return tostring(a):gsub("(%d+)(%.)",padnum) < tostring(b):gsub("(%d+)(%.)",padnum)
+	end)
+	for _,name in ipairs(keys) do
+		local value = tab[name]
+		name = encode(tostring(name), {["-"] = true, ["_"] = true, ["."] = true})
+		if key then
+			if M.options.cumulative_parameters and string.find(name, '^%d+$') then
+				name = tostring(key)
+			else
+				name = string.format('%s[%s]', tostring(key), tostring(name))
+			end
+		end
+		if type(value) == 'table' then
+			query[#query+1] = M.buildQuery(value, sep, name)
+		else
+			local value = encode(tostring(value), M.options.legal_in_query)
+			if value ~= "" then
+				query[#query+1] = string.format('%s=%s', name, value)
+			else
+				query[#query+1] = name
+			end
+		end
+	end
+	return table.concat(query, sep)
+end
+
+--- Parses the querystring to a table
+-- This function can parse multidimensional pairs and is mostly compatible
+-- with PHP usage of brackets in key names like ?param[key]=value
+-- @param str The querystring to parse
+-- @param sep The separator between key/value pairs, defaults to `&`
+-- @todo limit the max number of parameters with M.options.max_parameters
+-- @return a table representing the query key/value pairs
+function M.parseQuery(str, sep)
+	if not sep then
+		sep = M.options.separator or '&'
+	end
+
+	local values = {}
+	for key,val in str:gmatch(string.format('([^%q=]+)(=*[^%q=]*)', sep, sep)) do
+		local key = decodeValue(key)
+		local keys = {}
+		key = key:gsub('%[([^%]]*)%]', function(v)
+				-- extract keys between balanced brackets
+				if string.find(v, "^-?%d+$") then
+					v = tonumber(v)
+				else
+					v = decodeValue(v)
+				end
+				table.insert(keys, v)
+				return "="
+		end)
+		key = key:gsub('=+.*$', "")
+		key = key:gsub('%s', "_") -- remove spaces in parameter name
+		val = val:gsub('^=+', "")
+
+		if not values[key] then
+			values[key] = {}
+		end
+		if #keys > 0 and type(values[key]) ~= 'table' then
+			values[key] = {}
+		elseif #keys == 0 and type(values[key]) == 'table' then
+			values[key] = decodeValue(val)
+		elseif M.options.cumulative_parameters
+			and type(values[key]) == 'string' then
+			values[key] = { values[key] }
+			table.insert(values[key], decodeValue(val))
+		end
+
+		local t = values[key]
+		for i,k in ipairs(keys) do
+			if type(t) ~= 'table' then
+				t = {}
+			end
+			if k == "" then
+				k = #t+1
+			end
+			if not t[k] then
+				t[k] = {}
+			end
+			if i == #keys then
+				t[k] = val
+			end
+			t = t[k]
+		end
+
+	end
+	setmetatable(values, { __tostring = M.buildQuery })
+	return values
+end
+
+--- set the url query
+-- @param query Can be a string to parse or a table of key/value pairs
+-- @return a table representing the query key/value pairs
+function M:setQuery(query)
+	local query = query
+	if type(query) == 'table' then
+		query = M.buildQuery(query)
+	end
+	self.query = M.parseQuery(query)
+	return query
+end
+
+--- set the authority part of the url
+-- The authority is parsed to find the user, password, port and host if available.
+-- @param authority The string representing the authority
+-- @return a string with what remains after the authority was parsed
+function M:setAuthority(authority)
+	self.authority = authority
+	self.port = nil
+	self.host = nil
+	self.userinfo = nil
+	self.user = nil
+	self.password = nil
+
+	authority = authority:gsub('^([^@]*)@', function(v)
+		self.userinfo = v
+		return ''
+	end)
+
+	authority = authority:gsub(':(%d+)$', function(v)
+		self.port = tonumber(v)
+		return ''
+	end)
+
+	local function getIP(str)
+		-- ipv4
+		local chunks = { str:match("^(%d+)%.(%d+)%.(%d+)%.(%d+)$") }
+		if #chunks == 4 then
+			for _, v in pairs(chunks) do
+				if tonumber(v) > 255 then
+					return false
+				end
+			end
+			return str
+		end
+		-- ipv6
+		local chunks = { str:match("^%["..(("([a-fA-F0-9]*):"):rep(8):gsub(":$","%%]$"))) }
+		if #chunks == 8 or #chunks < 8 and
+			str:match('::') and not str:gsub("::", "", 1):match('::') then
+			for _,v in pairs(chunks) do
+				if #v > 0 and tonumber(v, 16) > 65535 then
+					return false
+				end
+			end
+			return str
+		end
+		return nil
+	end
+
+	local ip = getIP(authority)
+	if ip then
+		self.host = ip
+	elseif type(ip) == 'nil' then
+		-- domain
+		if authority ~= '' and not self.host then
+			local host = authority:lower()
+			if  string.match(host, '^[%d%a%-%.]+$') ~= nil and
+				string.sub(host, 0, 1) ~= '.' and
+				string.sub(host, -1) ~= '.' and
+				string.find(host, '%.%.') == nil then
+				self.host = host
+			end
+		end
+	end
+
+	if self.userinfo then
+		local userinfo = self.userinfo
+		userinfo = userinfo:gsub(':([^:]*)$', function(v)
+				self.password = v
+				return ''
+		end)
+		if string.find(userinfo, "^[%w%+%.]+$") then
+			self.user = userinfo
+		else
+			-- incorrect userinfo
+			self.userinfo = nil
+			self.user = nil
+			self.password = nil
+		end
+	end
+
+	return authority
+end
+
+--- Parse the url into the designated parts.
+-- Depending on the url, the following parts can be available:
+-- scheme, userinfo, user, password, authority, host, port, path,
+-- query, fragment
+-- @param url Url string
+-- @return a table with the different parts and a few other functions
+function M.parse(url)
+	local comp = {}
+	M.setAuthority(comp, "")
+	M.setQuery(comp, "")
+
+	local url = tostring(url or '')
+	url = url:gsub('#(.*)$', function(v)
+		comp.fragment = v
+		return ''
+	end)
+	url =url:gsub('^([%w][%w%+%-%.]*)%:', function(v)
+		comp.scheme = v:lower()
+		return ''
+	end)
+	url = url:gsub('%?(.*)', function(v)
+		M.setQuery(comp, v)
+		return ''
+	end)
+	url = url:gsub('^//([^/]*)', function(v)
+		M.setAuthority(comp, v)
+		return ''
+	end)
+
+	comp.path = url:gsub("([^/]+)", function (s) return encode(decode(s), M.options.legal_in_path) end)
+
+	setmetatable(comp, {
+		__index = M,
+		__tostring = M.build,
+		__concat = concat,
+		__div = M.addSegment
+	})
+	return comp
+end
+
+--- removes dots and slashes in urls when possible
+-- This function will also remove multiple slashes
+-- @param path The string representing the path to clean
+-- @return a string of the path without unnecessary dots and segments
+function M.removeDotSegments(path)
+	local fields = {}
+	if string.len(path) == 0 then
+		return ""
+	end
+	local startslash = false
+	local endslash = false
+	if string.sub(path, 1, 1) == "/" then
+		startslash = true
+	end
+	if (string.len(path) > 1 or startslash == false) and string.sub(path, -1) == "/" then
+		endslash = true
+	end
+
+	path:gsub('[^/]+', function(c) table.insert(fields, c) end)
+
+	local new = {}
+	local j = 0
+
+	for i,c in ipairs(fields) do
+		if c == '..' then
+			if j > 0 then
+				j = j - 1
+			end
+		elseif c ~= "." then
+			j = j + 1
+			new[j] = c
+		end
+	end
+	local ret = ""
+	if #new > 0 and j > 0 then
+		ret = table.concat(new, '/', 1, j)
+	else
+		ret = ""
+	end
+	if startslash then
+		ret = '/'..ret
+	end
+	if endslash then
+		ret = ret..'/'
+	end
+	return ret
+end
+
+local function reducePath(base_path, relative_path)
+	if string.sub(relative_path, 1, 1) == "/" then
+		return '/' .. string.gsub(relative_path, '^[%./]+', '')
+	end
+	local path = base_path
+	local startslash = string.sub(path, 1, 1) ~= "/";
+	if relative_path ~= "" then
+		path = (startslash and '' or '/') .. path:gsub("[^/]*$", "")
+	end
+	path = path .. relative_path
+	path = path:gsub("([^/]*%./)", function (s)
+		if s ~= "./" then return s else return "" end
+	end)
+	path = string.gsub(path, "/%.$", "/")
+	local reduced
+	while reduced ~= path do
+		reduced = path
+		path = string.gsub(reduced, "([^/]*/%.%./)", function (s)
+			if s ~= "../../" then return "" else return s end
+		end)
+	end
+	path = string.gsub(path, "([^/]*/%.%.?)$", function (s)
+		if s ~= "../.." then return "" else return s end
+	end)
+	local reduced
+	while reduced ~= path do
+		reduced = path
+		path = string.gsub(reduced, '^/?%.%./', '')
+	end
+	return (startslash and '' or '/') .. path
+end
+
+--- builds a new url by using the one given as parameter and resolving paths
+-- @param other A string or a table representing a url
+-- @return a new url table
+function M:resolve(other)
+	if type(self) == "string" then
+		self = M.parse(self)
+	end
+	if type(other) == "string" then
+		other = M.parse(other)
+	end
+	if other.scheme then
+		return other
+	else
+		other.scheme = self.scheme
+		if not other.authority or other.authority == "" then
+			other:setAuthority(self.authority)
+			if not other.path or other.path == "" then
+				other.path = self.path
+				local query = other.query
+				if not query or not next(query) then
+					other.query = self.query
+				end
+			else
+				other.path = reducePath(self.path, other.path)
+			end
+		end
+		return other
+	end
+end
+
+--- normalize a url path following some common normalization rules
+-- described on <a href="http://en.wikipedia.org/wiki/URL_normalization">The URL normalization page of Wikipedia</a>
+-- @return the normalized path
+function M:normalize()
+	if type(self) == 'string' then
+		self = M.parse(self)
+	end
+	if self.path then
+		local path = self.path
+		path = reducePath(path, "")
+		-- normalize multiple slashes
+		path = string.gsub(path, "//+", "/")
+		self.path = path
+	end
+	return self
+end
+
+return M

lib/plugins/crowdsec/utils.lua

@@ -13,6 +13,7 @@ M.HTTP_CODE["401"] = ngx.HTTP_UNAUTHORIZED
 M.HTTP_CODE["403"] = ngx.HTTP_FORBIDDEN
 M.HTTP_CODE["404"] = ngx.HTTP_NOT_FOUND
 M.HTTP_CODE["405"] = ngx.HTTP_NOT_ALLOWED
+M.HTTP_CODE["406"] = ngx.HTTP_NOT_ACCEPTABLE
 M.HTTP_CODE["500"] = ngx.HTTP_INTERNAL_SERVER_ERROR
 
 function M.read_file(path)