feat: load balancing and cooldowns (merge pull request #19 from davidovski/load_balancing)

Load balancing and Cooldowns

README.md

@@ -31,7 +31,7 @@ You can access the full list of LibreX and LibreY instances on one of the follow
 ### About LibreY
 
 LibreY gives you text results from DuckDuckGo or Google, images from Qwant, and torrents from i.e. Ahmia and popular torrent sites without spying on you.
-<br>LibreY doesn't save any type of data about the user, there are no logs (except NGINX logs if the host sets them), no caches.
+<br>LibreY doesn't save **any** type of data about the user, there are no logs (except NGINX logs if the host sets them).
 
 ### LibreY compared to other metasearch engines
 

config.php.example

@@ -17,8 +17,11 @@
         "disable_hidden_service_search" => false,
 
         // Fallback to another librex instance if google search fails
-        // This may greatly increase the time it takes to get a result and in some cases results in 504 errors
+        // This may greatly increase the time it takes to get a result, if a direct search is not possible
-        "instance_fallback" => false,
+        "instance_fallback" => true,
+
+        // how long in minutes to put google/other instances on cooldown if they aren't responding
+        "request_cooldown" => 25,
 
         /*
             Preset privacy friendly frontends for users, these can be overwritten by users in the settings
@@ -148,7 +151,8 @@
             CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP,
             CURLOPT_MAXREDIRS => 5,
             CURLOPT_TIMEOUT => 3,
-            CURLOPT_VERBOSE => false
+            CURLOPT_VERBOSE => false,
+            CURLOPT_FOLLOWLOCATION => true
         )
     );
 ?>

docker/attributes.sh

@@ -23,7 +23,7 @@ export CONFIG_GOOGLE_DOMAIN="${CONFIG_GOOGLE_DOMAIN:-"com"}"
 export CONFIG_GOOGLE_LANGUAGE_SITE="${CONFIG_GOOGLE_LANGUAGE_SITE:-"en"}"
 export CONFIG_GOOGLE_LANGUAGE_RESULTS="${CONFIG_GOOGLE_LANGUAGE_RESULTS:-"en"}"
 export CONFIG_GOOGLE_NUMBER_OF_RESULTS="${CONFIG_GOOGLE_NUMBER_OF_RESULTS:-"10"}"
-export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK}:-true}
+export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK}:-true}"
 export CONFIG_INVIDIOUS_INSTANCE="${CONFIG_INVIDIOUS_INSTANCE:-"invidious.snopyta.org"}"
 export CONFIG_HIDDEN_SERVICE_SEARCH=${CONFIG_HIDDEN_SERVICE_SEARCH:-false}
 export CONFIG_DISABLE_BITTORRENT_SEARCH=${CONFIG_DISABLE_BITTORRENT_SEARCH:-false}

docker/php/php.dockerfile

@@ -56,7 +56,7 @@ ENV CURLOPT_VERBOSE=true
 
 # Install PHP-FPM using Alpine's package manager, apk
 # Configure PHP-FPM to listen on a Unix socket instead of a TCP port, which is more secure and efficient
-RUN apk add php8 php8-fpm php8-dom php8-curl php8-json --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing &&\
+RUN apk add php8 php8-fpm php8-dom php8-curl php8-json php8-apcu --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing &&\
     sed -i 's/^\s*listen = 127.0.0.1:9000/listen = \/run\/php8\/php-fpm8.sock/' ${WWW_CONFIG} &&\
     sed -i 's/^\s*;\s*listen.owner = nobody/listen.owner = nginx/' ${WWW_CONFIG} &&\
     sed -i 's/^\s*;\s*listen.group = nobody/listen.group = nginx/' ${WWW_CONFIG} &&\

engines/librex/fallback.php

@@ -20,20 +20,24 @@
         }
     }
 
-
+    function load_instances($cooldowns) {
-    function get_librex_results($opts) {
-        if (!$opts->do_fallback)
-            return array();
-
         $instances_json = json_decode(file_get_contents("instances.json"), true);
 
         if (empty($instances_json["instances"]))
             return array();
 
-        // TODO pick instances which aren't on cooldown
-
         $instances = array_map(fn($n) => $n['clearnet'], array_filter($instances_json['instances'], fn($n) => !is_null($n['clearnet'])));
+        $instances = array_filter($instances, fn($n) => !has_cooldown($n, $cooldowns));
         shuffle($instances);
+        return $instances;
+    }
+
+    function get_librex_results($opts) {
+        if (!$opts->do_fallback)
+            return array();
+
+        $cooldowns = $opts->cooldowns;
+        $instances = load_instances($cooldowns);
 
         $results = array();
         $tries = 0;
@@ -52,12 +56,13 @@
             if (count($results) > 1)
                 return $results;
 
+            // on fail then do this
+            $timeout = ($opts->request_cooldown ?? "1") * 60;
+            $cooldowns = set_cooldown($instance, $timeout, $cooldowns);
+
         } while (!empty($instances));
 
-        if (empty($instances))
         return array();
-
-        return array_values($results);
     }
 
 ?>

engines/text/google.php

@@ -1,7 +1,6 @@
 <?php
     class GoogleRequest extends EngineRequest {
-        function get_request_url()
+        public function get_request_url() {
-        {
 
             $query_encoded = str_replace("%22", "\"", urlencode($this->query));
             $results = array();

engines/text/text.php

@@ -5,19 +5,23 @@
             $this->page = $opts->page;
             $this->opts = $opts;
 
-            $engine = $opts->preferred_engines["text"] ?? "google";
+            $this->engine = $opts->preferred_engines["text"] ?? "google";
 
             $query_parts = explode(" ", $this->query);
             $last_word_query = end($query_parts);
             if (substr($this->query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!")
                 check_ddg_bang($this->query, $opts);
 
-            if ($engine == "google") {
+            if (has_cooldown($this->engine, $this->opts->cooldowns))
+                return;
+
+            if ($this->engine == "google") {
+                
                 require "engines/text/google.php";
                 $this->engine_request = new GoogleRequest($opts, $mh);
             }
 
-            if ($engine == "duckduckgo") {
+            if ($this->engine == "duckduckgo") {
                 require "engines/text/duckduckgo.php";
                 $this->engine_request = new DuckDuckGoRequest($opts, $mh);
             }
@@ -27,6 +31,9 @@
         }
 
         public function get_results() {
+            if (!$this->engine_request)
+                return array();
+
             $results = $this->engine_request->get_results();
 
             if ($this->special_request) {
@@ -36,6 +43,9 @@
                     $results = array_merge(array($special_result), $results);
             }
 
+            if (count($results) <= 1)
+                set_cooldown($this->engine, ($opts->request_cooldown ?? "1") * 60, $this->opts->cooldowns);
+
             return $results;
         }
 

misc/cooldowns.php

@@ -0,0 +1,22 @@
+<?php
+    function load_cooldowns() {
+        if (function_exists("apcu_fetch"))
+            return apcu_exists("cooldowns") ? apcu_fetch("cooldowns") : array();
+        return array();
+    }
+
+    function save_cooldowns($cooldowns) {
+        if (function_exists("apcu_store"))
+            apcu_store("cooldowns", $cooldowns);
+    }
+
+    function set_cooldown($instance, $timeout, $cooldowns) {
+        $cooldowns[$instance] = time() + $timeout;
+        save_cooldowns($cooldowns);
+        return $cooldowns;
+    }
+
+    function has_cooldown($instance, $cooldowns) {
+        return ($cooldowns[$instance] ?? 0) > time();
+    }
+?>

misc/search_engine.php

@@ -33,6 +33,8 @@
     function load_opts() {
         $opts = require "config.php";
 
+        $opts->request_cooldown ??= 25;
+
         $opts->query = trim($_REQUEST["q"] ?? "");
         $opts->type = (int) ($_REQUEST["t"] ?? 0);
         $opts->page = (int) ($_REQUEST["p"] ?? 0);
@@ -45,7 +47,7 @@
 
         $opts->disable_frontends = (int) ($_REQUEST["nf"] ?? 0) == 1 || isset($_COOKIE["disable_frontends"]);
 
-        $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? ""));
+        $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? $opts->language));
 
         $opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0;
         if (!$opts->instance_fallback) {
@@ -57,6 +59,7 @@
         foreach (array_keys($opts->frontends ?? array()) as $frontend) {
             $opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? $opts->frontends[$frontend]["instance_url"];
         }
+
         return $opts;
     }
 
@@ -110,6 +113,9 @@
     }
 
     function fetch_search_results($opts, $do_print) {
+        require "misc/cooldowns.php";
+        $opts->cooldowns = load_cooldowns();
+
         $start_time = microtime(true);
         $mh = curl_multi_init();
         $search_category = init_search($opts, $mh);