Fijxu/librex
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: load balancing and cooldowns (merge pull request #19 from davidovski/load_balancing)

Browse source 
Load balancing and Cooldowns
This commit is contained in:
Ahwx 2023-08-26 11:06:24 +02:00 committed by GitHub
commit 3ae47a1cfc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 70 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -31,7 +31,7 @@ You can access the full list of LibreX and LibreY instances on one of the follow
### About LibreY
LibreY gives you text results from DuckDuckGo or Google, images from Qwant, and torrents from i.e. Ahmia and popular torrent sites without spying on you.
<br>LibreY doesn't save any type of data about the user, there are no logs (except NGINX logs if the host sets them), no caches.
<br>LibreY doesn't save **any** type of data about the user, there are no logs (except NGINX logs if the host sets them).
### LibreY compared to other metasearch engines

10
config.php.example
View file

@ -17,8 +17,11 @@
"disable_hidden_service_search" => false,
// Fallback to another librex instance if google search fails
// This may greatly increase the time it takes to get a result and in some cases results in 504 errors
"instance_fallback" => false,
// This may greatly increase the time it takes to get a result, if a direct search is not possible
"instance_fallback" => true,
// how long in minutes to put google/other instances on cooldown if they aren't responding
"request_cooldown" => 25,
/*
Preset privacy friendly frontends for users, these can be overwritten by users in the settings
@ -148,7 +151,8 @@
CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP,
CURLOPT_MAXREDIRS => 5,
CURLOPT_TIMEOUT => 3,
CURLOPT_VERBOSE => false
CURLOPT_VERBOSE => false,
CURLOPT_FOLLOWLOCATION => true
)
);
?>

2
docker/attributes.sh
View file

@ -23,7 +23,7 @@ export CONFIG_GOOGLE_DOMAIN="${CONFIG_GOOGLE_DOMAIN:-"com"}"
export CONFIG_GOOGLE_LANGUAGE_SITE="${CONFIG_GOOGLE_LANGUAGE_SITE:-"en"}"
export CONFIG_GOOGLE_LANGUAGE_RESULTS="${CONFIG_GOOGLE_LANGUAGE_RESULTS:-"en"}"
export CONFIG_GOOGLE_NUMBER_OF_RESULTS="${CONFIG_GOOGLE_NUMBER_OF_RESULTS:-"10"}"
export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK}:-true}
export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK}:-true}"
export CONFIG_INVIDIOUS_INSTANCE="${CONFIG_INVIDIOUS_INSTANCE:-"invidious.snopyta.org"}"
export CONFIG_HIDDEN_SERVICE_SEARCH=${CONFIG_HIDDEN_SERVICE_SEARCH:-false}
export CONFIG_DISABLE_BITTORRENT_SEARCH=${CONFIG_DISABLE_BITTORRENT_SEARCH:-false}

2
docker/php/php.dockerfile
View file

@ -56,7 +56,7 @@ ENV CURLOPT_VERBOSE=true
# Install PHP-FPM using Alpine's package manager, apk
# Configure PHP-FPM to listen on a Unix socket instead of a TCP port, which is more secure and efficient
RUN apk add php8 php8-fpm php8-dom php8-curl php8-json --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing &&\
RUN apk add php8 php8-fpm php8-dom php8-curl php8-json php8-apcu --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing &&\
sed -i 's/^\s*listen = 127.0.0.1:9000/listen = \/run\/php8\/php-fpm8.sock/' ${WWW_CONFIG} &&\
sed -i 's/^\s*;\s*listen.owner = nobody/listen.owner = nginx/' ${WWW_CONFIG} &&\
sed -i 's/^\s*;\s*listen.group = nobody/listen.group = nginx/' ${WWW_CONFIG} &&\

29
engines/librex/fallback.php
View file

@ -15,25 +15,29 @@
$response = json_decode($response, true);
if (!$response)
return array();
return array_values($response);
}
}
function get_librex_results($opts) {
if (!$opts->do_fallback)
return array();
function load_instances($cooldowns) {
$instances_json = json_decode(file_get_contents("instances.json"), true);
if (empty($instances_json["instances"]))
return array();
// TODO pick instances which aren't on cooldown
$instances = array_map(fn($n) => $n['clearnet'], array_filter($instances_json['instances'], fn($n) => !is_null($n['clearnet'])));
$instances = array_filter($instances, fn($n) => !has_cooldown($n, $cooldowns));
shuffle($instances);
return $instances;
}
function get_librex_results($opts) {
if (!$opts->do_fallback)
return array();
$cooldowns = $opts->cooldowns;
$instances = load_instances($cooldowns);
$results = array();
$tries = 0;
@ -52,12 +56,13 @@
if (count($results) > 1)
return $results;
} while ( !empty($instances));
// on fail then do this
$timeout = ($opts->request_cooldown ?? "1") * 60;
$cooldowns = set_cooldown($instance, $timeout, $cooldowns);
if (empty($instances))
return array();
} while (!empty($instances));
return array_values($results);
return array();
}
?>

3
engines/text/google.php
View file

@ -1,7 +1,6 @@
<?php
class GoogleRequest extends EngineRequest {
function get_request_url()
{
public function get_request_url() {
$query_encoded = str_replace("%22", "\"", urlencode($this->query));
$results = array();

16
engines/text/text.php
View file

@ -5,19 +5,23 @@
$this->page = $opts->page;
$this->opts = $opts;
$engine = $opts->preferred_engines["text"] ?? "google";
$this->engine = $opts->preferred_engines["text"] ?? "google";
$query_parts = explode(" ", $this->query);
$last_word_query = end($query_parts);
if (substr($this->query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!")
check_ddg_bang($this->query, $opts);
if ($engine == "google") {
if (has_cooldown($this->engine, $this->opts->cooldowns))
return;
if ($this->engine == "google") {
require "engines/text/google.php";
$this->engine_request = new GoogleRequest($opts, $mh);
}
if ($engine == "duckduckgo") {
if ($this->engine == "duckduckgo") {
require "engines/text/duckduckgo.php";
$this->engine_request = new DuckDuckGoRequest($opts, $mh);
}
@ -27,6 +31,9 @@
}
public function get_results() {
if (!$this->engine_request)
return array();
$results = $this->engine_request->get_results();
if ($this->special_request) {
@ -36,6 +43,9 @@
$results = array_merge(array($special_result), $results);
}
if (count($results) <= 1)
set_cooldown($this->engine, ($opts->request_cooldown ?? "1") * 60, $this->opts->cooldowns);
return $results;
}

22
misc/cooldowns.php Normal file
View file

@ -0,0 +1,22 @@
<?php
function load_cooldowns() {
if (function_exists("apcu_fetch"))
return apcu_exists("cooldowns") ? apcu_fetch("cooldowns") : array();
return array();
}
function save_cooldowns($cooldowns) {
if (function_exists("apcu_store"))
apcu_store("cooldowns", $cooldowns);
}
function set_cooldown($instance, $timeout, $cooldowns) {
$cooldowns[$instance] = time() + $timeout;
save_cooldowns($cooldowns);
return $cooldowns;
}
function has_cooldown($instance, $cooldowns) {
return ($cooldowns[$instance] ?? 0) > time();
}
?>

8
misc/search_engine.php
View file

@ -33,6 +33,8 @@
function load_opts() {
$opts = require "config.php";
$opts->request_cooldown ??= 25;
$opts->query = trim($_REQUEST["q"] ?? "");
$opts->type = (int) ($_REQUEST["t"] ?? 0);
$opts->page = (int) ($_REQUEST["p"] ?? 0);
@ -45,7 +47,7 @@
$opts->disable_frontends = (int) ($_REQUEST["nf"] ?? 0) == 1 || isset($_COOKIE["disable_frontends"]);
$opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? ""));
$opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? $opts->language));
$opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0;
if (!$opts->instance_fallback) {
@ -57,6 +59,7 @@
foreach (array_keys($opts->frontends ?? array()) as $frontend) {
$opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? $opts->frontends[$frontend]["instance_url"];
}
return $opts;
}
@ -110,6 +113,9 @@
}
function fetch_search_results($opts, $do_print) {
require "misc/cooldowns.php";
$opts->cooldowns = load_cooldowns();
$start_time = microtime(true);
$mh = curl_multi_init();
$search_category = init_search($opts, $mh);