From fd93a0e1541832cb450580fe902b7dc47fc6e96b Mon Sep 17 00:00:00 2001 From: davidovski Date: Mon, 21 Aug 2023 16:29:50 +0100 Subject: [PATCH 01/40] Add EngineRequest class --- engines/ahmia/hidden_service.php | 23 --- engines/bittorrent/merge.php | 6 +- engines/duckduckgo/text.php | 201 ++++------------------- engines/google/text.php | 269 +++++++------------------------ engines/special.php | 88 ++++++++++ engines/special/currency.php | 11 +- engines/special/definition.php | 12 +- engines/special/ip.php | 22 ++- engines/special/tor.php | 18 ++- engines/special/user_agent.php | 10 +- engines/special/weather.php | 39 +++-- engines/special/wikipedia.php | 28 ++-- engines/text.php | 143 ++++++++++++++++ misc/tools.php | 80 --------- search.php | 38 +++-- 15 files changed, 447 insertions(+), 541 deletions(-) create mode 100644 engines/special.php create mode 100644 engines/text.php diff --git a/engines/ahmia/hidden_service.php b/engines/ahmia/hidden_service.php index 5a840f7..d501b96 100644 --- a/engines/ahmia/hidden_service.php +++ b/engines/ahmia/hidden_service.php @@ -27,27 +27,4 @@ return $results; } - - function print_hidden_service_results($results) - { - echo "
"; - - foreach($results as $result) - { - $title = $result["title"]; - $url = $result["url"]; - $base_url = $result["base_url"]; - $description = $result["description"]; - - echo "
"; - echo ""; - echo "$base_url"; - echo "

$title

"; - echo ""; - echo "$description"; - echo "
"; - } - - echo "
"; - } ?> diff --git a/engines/bittorrent/merge.php b/engines/bittorrent/merge.php index 6e105f1..1106ba3 100644 --- a/engines/bittorrent/merge.php +++ b/engines/bittorrent/merge.php @@ -23,7 +23,7 @@ $_1337x_url, $sukebei_url ); - + $mh = curl_multi_init(); $chs = $results = array(); @@ -32,7 +32,7 @@ $ch = curl_init($url); curl_setopt_array($ch, $config->curl_settings); array_push($chs, $ch); - curl_multi_add_handle($mh, $ch); + curl_multi_add_handle($mh, $ch); } $running = null; @@ -69,7 +69,7 @@ break; } } - + $seeders = array_column($results, "seeders"); array_multisort($seeders, SORT_DESC, $results); diff --git a/engines/duckduckgo/text.php b/engines/duckduckgo/text.php index 2bbab7f..e4bdbac 100644 --- a/engines/duckduckgo/text.php +++ b/engines/duckduckgo/text.php @@ -1,10 +1,9 @@ query)); $results = array(); // $domain = $config->google_domain; @@ -25,180 +24,46 @@ if (isset($_COOKIE["safe_search"])) $url .= "&safe=medium"; + return $url; + } - $google_ch = curl_init($url); - curl_setopt_array($google_ch, $config->curl_settings); - curl_multi_add_handle($mh, $google_ch); - - $special_search = $page ? 0 : check_for_special_search($query); - $special_ch = null; - $url = null; - if ($special_search != 0) - { - switch ($special_search) - { - case 1: - $url = "https://cdn.moneyconvert.net/api/latest.json"; - break; - case 2: - $split_query = explode(" ", $query); - $reversed_split_q = array_reverse($split_query); - $word_to_define = $reversed_split_q[1]; - $url = "https://api.dictionaryapi.dev/api/v2/entries/en/$word_to_define"; - break; - case 5: - $url = "https://wttr.in/@" . $_SERVER["REMOTE_ADDR"] . "?format=j1"; - break; - case 6: - $url = "https://check.torproject.org/torbulkexitlist"; - break; - case 7: - $wikipedia_language = isset($_COOKIE["wikipedia_language"]) ? trim(htmlspecialchars($_COOKIE["wikipedia_language"])) : $config->wikipedia_language; - if (in_array($wikipedia_language, json_decode(file_get_contents("static/misc/wikipedia_langs.json"), true))) - $url = "https://$wikipedia_language.wikipedia.org/w/api.php?format=json&action=query&prop=extracts%7Cpageimages&exintro&explaintext&redirects=1&pithumbsize=500&titles=$query_encoded"; - break; - } + public function get_results() { + $results = array(); + $xpath = get_xpath(curl_multi_getcontent($this->ch)); - if ($url != NULL) + foreach($xpath->query("/html/body/div[1]/div[". count($xpath->query('/html/body/div[1]/div')) ."]/div/div/div/div") as $result) { - $special_ch = curl_init($url); - curl_setopt_array($special_ch, $config->curl_settings); - curl_multi_add_handle($mh, $special_ch); - } - } + $url = $xpath->evaluate(".//h2[@class='result__title']//a/@href", $result)[0]; + + if ($url == null) + continue; - $running = null; - do { - curl_multi_exec($mh, $running); - } while ($running); - - if (curl_getinfo($google_ch)['http_code'] != '200') - { - require "engines/librex/text.php"; - return get_librex_results($query, $page); - } - - - - if ($special_search != 0) - { - $special_result = null; - - switch ($special_search) - { - case 1: - require "engines/special/currency.php"; - $special_result = currency_results($query, curl_multi_getcontent($special_ch)); - break; - case 2: - require "engines/special/definition.php"; - $special_result = definition_results($query, curl_multi_getcontent($special_ch)); - break; - - case 3: - require "engines/special/ip.php"; - $special_result = ip_result(); - break; - case 4: - require "engines/special/user_agent.php"; - $special_result = user_agent_result(); - break; - case 5: - require "engines/special/weather.php"; - $special_result = weather_results(curl_multi_getcontent($special_ch)); - break; - case 6: - require "engines/special/tor.php"; - $special_result = tor_result(curl_multi_getcontent($special_ch)); - break; - case 7: - require "engines/special/wikipedia.php"; - $special_result = wikipedia_results($query, curl_multi_getcontent($special_ch)); - break; - } - - if ($special_result != null) - array_push($results, $special_result); - } - - $xpath = get_xpath(curl_multi_getcontent($google_ch)); - - foreach($xpath->query("/html/body/div[1]/div[". count($xpath->query('/html/body/div[1]/div')) ."]/div/div/div/div") as $result) - { - $url = $xpath->evaluate(".//h2[@class='result__title']//a/@href", $result)[0]; - - if ($url == null) - continue; - - if (!empty($results)) // filter duplicate results, ignore special result - { - if (!array_key_exists("special_response", end($results))) + if (!empty($results)) // filter duplicate results + { if (end($results)["url"] == $url->textContent) continue; - } + } - $url = $url->textContent; + $url = $url->textContent; - $url = check_for_privacy_frontend($url); + $url = check_for_privacy_frontend($url); - $title = $xpath->evaluate(".//h2[@class='result__title']", $result)[0]; - $description = $xpath->evaluate(".//a[@class='result__snippet']", $result)[0]; - - array_push($results, - array ( - "title" => htmlspecialchars($title->textContent), - "url" => htmlspecialchars($url), - "base_url" => htmlspecialchars(get_base_url($url)), - "description" => $description == null ? - "No description was provided for this site." : - htmlspecialchars($description->textContent) - ) - ); - } + $title = $xpath->evaluate(".//h2[@class='result__title']", $result)[0]; + $description = $xpath->evaluate(".//a[@class='result__snippet']", $result)[0]; + array_push($results, + array ( + "title" => htmlspecialchars($title->textContent), + "url" => htmlspecialchars($url), + "base_url" => htmlspecialchars(get_base_url($url)), + "description" => $description == null ? + "No description was provided for this site." : + htmlspecialchars($description->textContent) + ) + ); + } return $results; - } - - function print_text_results($results) - { - $special = $results[0]; - if (array_key_exists("special_response", $special)) - { - $response = $special["special_response"]["response"]; - $source = $special["special_response"]["source"]; - - echo "

"; - if (array_key_exists("image", $special["special_response"])) - { - $image_url = $special["special_response"]["image"]; - echo ""; - } - echo $response; - if ($source) - echo "$source"; - echo "

"; - - array_shift($results); } - echo "
"; - - foreach($results as $result) - { - $title = $result["title"]; - $url = $result["url"]; - $base_url = $result["base_url"]; - $description = $result["description"]; - - echo "
"; - echo ""; - echo "$base_url"; - echo "

$title

"; - echo ""; - echo "$description"; - echo "
"; - } - - echo "
"; } ?> diff --git a/engines/google/text.php b/engines/google/text.php index 75f9082..323c0e7 100644 --- a/engines/google/text.php +++ b/engines/google/text.php @@ -1,226 +1,81 @@ google_domain; - $site_language = isset($_COOKIE["google_language_site"]) ? trim(htmlspecialchars($_COOKIE["google_language_site"])) : $config->google_language_site; - $results_language = isset($_COOKIE["google_language_results"]) ? trim(htmlspecialchars($_COOKIE["google_language_results"])) : $config->google_language_results; - $number_of_results = isset($_COOKIE["google_number_of_results"]) ? trim(htmlspecialchars($_COOKIE["google_number_of_results"])) : $config->google_number_of_results; - - $url = "https://www.google.$domain/search?q=$query_encoded&nfpr=1&start=$page"; - error_log($url); - - if (3 > strlen($site_language) && 0 < strlen($site_language)) - $url .= "&hl=$site_language"; - - if (3 > strlen($results_language) && 0 < strlen($results_language)) - $url .= "&lr=lang_$results_language"; - - if (3 > strlen($number_of_results) && 0 < strlen($number_of_results)) - $url .= "&num=$number_of_results"; - - if (isset($_COOKIE["safe_search"])) - $url .= "&safe=medium"; - - $google_ch = curl_init($url); - curl_setopt_array($google_ch, $config->curl_settings); - curl_multi_add_handle($mh, $google_ch); - - $special_search = $page ? 0 : check_for_special_search($query); - $special_ch = null; - $url = null; - if ($special_search != 0) + class GoogleRequest extends EngineRequest { + function get_request_url() { - switch ($special_search) - { - case 1: - $url = "https://cdn.moneyconvert.net/api/latest.json"; - break; - case 2: - $split_query = explode(" ", $query); - $reversed_split_q = array_reverse($split_query); - $word_to_define = $reversed_split_q[1]; - $url = "https://api.dictionaryapi.dev/api/v2/entries/en/$word_to_define"; - break; - case 5: - $url = "https://wttr.in/@" . $_SERVER["REMOTE_ADDR"] . "?format=j1"; - break; - case 6: - $url = "https://check.torproject.org/torbulkexitlist"; - break; - case 7: - $wikipedia_language = isset($_COOKIE["wikipedia_language"]) ? trim(htmlspecialchars($_COOKIE["wikipedia_language"])) : $config->wikipedia_language; - if (in_array($wikipedia_language, json_decode(file_get_contents("static/misc/wikipedia_langs.json"), true))) - $url = "https://$wikipedia_language.wikipedia.org/w/api.php?format=json&action=query&prop=extracts%7Cpageimages&exintro&explaintext&redirects=1&pithumbsize=500&titles=$query_encoded"; - break; - } + $page = isset($_REQUEST["p"]) ? (int) $_REQUEST["p"] : 0; - if ($url != NULL) - { - $special_ch = curl_init($url); - curl_setopt_array($special_ch, $config->curl_settings); - curl_multi_add_handle($mh, $special_ch); - } - } + $query_encoded = str_replace("%22", "\"", urlencode($this->query)); + $results = array(); - $running = null; - do { - curl_multi_exec($mh, $running); - } while ($running); + $domain = $this->config->google_domain; + $site_language = isset($_COOKIE["google_language_site"]) ? trim(htmlspecialchars($_COOKIE["google_language_site"])) : $this->config->google_language_site; + $results_language = isset($_COOKIE["google_language_results"]) ? trim(htmlspecialchars($_COOKIE["google_language_results"])) : $this->config->google_language_results; + $number_of_results = isset($_COOKIE["google_number_of_results"]) ? trim(htmlspecialchars($_COOKIE["google_number_of_results"])) : $this->config->google_number_of_results; - if (curl_getinfo($google_ch)['http_code'] != '200') - { - require "engines/librex/text.php"; - return get_librex_results($query, $page); + $url = "https://www.google.$domain/search?q=$query_encoded&nfpr=1&start=$page"; + error_log($url); + + if (3 > strlen($site_language) && 0 < strlen($site_language)) + $url .= "&hl=$site_language"; + + if (3 > strlen($results_language) && 0 < strlen($results_language)) + $url .= "&lr=lang_$results_language"; + + if (3 > strlen($number_of_results) && 0 < strlen($number_of_results)) + $url .= "&num=$number_of_results"; + + if (isset($_COOKIE["safe_search"])) + $url .= "&safe=medium"; + + return $url; } - $special_result = array(); - if ($special_search != 0) - { - switch ($special_search) + public function get_results() { + $results = array(); + $xpath = get_xpath(curl_multi_getcontent($this->ch)); + + $didyoumean = $xpath->query(".//a[@class='gL9Hy']")[0]; + + if (!is_null($didyoumean)) + array_push($results, array( + "did_you_mean" => $didyoumean->textContent + )); + + foreach($xpath->query("//div[@id='search']//div[contains(@class, 'g')]") as $result) { - case 1: - require "engines/special/currency.php"; - $special_result = currency_results($query, curl_multi_getcontent($special_ch)); - break; - case 2: - require "engines/special/definition.php"; - $special_result = definition_results($query, curl_multi_getcontent($special_ch)); - break; + $url = $xpath->evaluate(".//div[@class='yuRUbf']//a/@href", $result)[0]; - case 3: - require "engines/special/ip.php"; - $special_result = ip_result(); - break; - case 4: - require "engines/special/user_agent.php"; - $special_result = user_agent_result(); - break; - case 5: - require "engines/special/weather.php"; - $special_result = weather_results(curl_multi_getcontent($special_ch)); - break; - case 6: - require "engines/special/tor.php"; - $special_result = tor_result(curl_multi_getcontent($special_ch)); - break; - case 7: - require "engines/special/wikipedia.php"; - $special_result = wikipedia_results($query, curl_multi_getcontent($special_ch)); - break; - } - } + if ($url == null) + continue; - $xpath = get_xpath(curl_multi_getcontent($google_ch)); - - $didyoumean = $xpath->query(".//a[@class='gL9Hy']")[0]; - - if (!is_null($didyoumean)) - $special_result["did_you_mean"] = $didyoumean->textContent; - - if (!empty($special_result)) - array_push($results, $special_result); - - - - foreach($xpath->query("//div[@id='search']//div[contains(@class, 'g')]") as $result) - { - $url = $xpath->evaluate(".//div[@class='yuRUbf']//a/@href", $result)[0]; - - if ($url == null) - continue; - - if (!empty($results)) // filter duplicate results, ignore special result - { - if (!array_key_exists("special_response", end($results))) + if (!empty($results)) // filter duplicate results, ignore special result + { if (end($results)["url"] == $url->textContent) continue; + } + + $url = $url->textContent; + + $url = check_for_privacy_frontend($url); + + $title = $xpath->evaluate(".//h3", $result)[0]; + $description = $xpath->evaluate(".//div[contains(@class, 'VwiC3b')]", $result)[0]; + + array_push($results, + array ( + "title" => htmlspecialchars($title->textContent), + "url" => htmlspecialchars($url), + "base_url" => htmlspecialchars(get_base_url($url)), + "description" => $description == null ? + "No description was provided for this site." : + htmlspecialchars($description->textContent) + ) + ); } - $url = $url->textContent; - - $url = check_for_privacy_frontend($url); - - $title = $xpath->evaluate(".//h3", $result)[0]; - $description = $xpath->evaluate(".//div[contains(@class, 'VwiC3b')]", $result)[0]; - - array_push($results, - array ( - "title" => htmlspecialchars($title->textContent), - "url" => htmlspecialchars($url), - "base_url" => htmlspecialchars(get_base_url($url)), - "description" => $description == null ? - "No description was provided for this site." : - htmlspecialchars($description->textContent) - ) - ); + return $results; } - - return $results; - } - - function print_text_results($results) - { - - if (empty($results)) - return; - - $special = $results[0]; - - if (array_key_exists("did_you_mean", $special)) - { - $didyoumean = $special["did_you_mean"]; - $new_url = "/search.php?q=" . urlencode($didyoumean); - echo "

Did you mean "; - echo "$didyoumean"; - echo "?

"; - } - - if (array_key_exists("special_response", $special)) - { - $response = $special["special_response"]["response"]; - $source = $special["special_response"]["source"]; - - echo "

"; - if (array_key_exists("image", $special["special_response"])) - { - $image_url = $special["special_response"]["image"]; - echo ""; - } - echo $response; - if ($source) - echo "$source"; - echo "

"; - } - - echo "
"; - - foreach($results as $result) - { - if (!array_key_exists("title", $result)) - continue; - - $title = $result["title"]; - $url = $result["url"]; - $base_url = $result["base_url"]; - $description = $result["description"]; - - echo "
"; - echo ""; - echo "$base_url"; - echo "

$title

"; - echo ""; - echo "$description"; - echo "
"; - } - - echo "
"; } ?> diff --git a/engines/special.php b/engines/special.php new file mode 100644 index 0000000..e453057 --- /dev/null +++ b/engines/special.php @@ -0,0 +1,88 @@ += 4) // currency + { + $amount_to_convert = floatval($split_query[0]); + if ($amount_to_convert != 0) + return 1; + } + else if (strpos($query_lower, "mean") && count($split_query) >= 2) // definition + { + return 2; + } + else if (strpos($query_lower, "my") !== false) + { + if (strpos($query_lower, "ip")) + { + return 3; + } + else if (strpos($query_lower, "user agent") || strpos($query_lower, "ua")) + { + return 4; + } + } + else if (strpos($query_lower, "weather") !== false) + { + return 5; + } + else if ($query_lower == "tor") + { + return 6; + } + else if (3 > count(explode(" ", $query))) // wikipedia + { + return 7; + } + + return 0; + } + + function get_special_search_request ($query, $page, $mh, $config) { + $special_search = $page ? 0 : check_for_special_search($query); + $special_request = null; + $url = null; + + if ($special_search == 0) + return null; + + switch ($special_search) { + case 1: + require "engines/special/currency.php"; + $special_request = new CurrencyRequest($query, $mh, $config); + break; + case 2: + require "engines/special/definition.php"; + $special_request = new DefinitionRequest($query, $mh, $config); + break; + case 3: + require "engines/special/ip.php"; + $special_request = new IPRequest($query, $mh, $config); + break; + case 4: + require "engines/special/user_agent.php"; + $special_request = new UserAgentRequest($query, $mh, $config); + break; + case 5: + require "engines/special/weather.php"; + $special_request = new WeatherRequest($query, $mh, $config); + break; + case 6: + require "engines/special/tor.php"; + $special_request = new TorRequest($query, $mh, $config); + break; + case 7: + require "engines/special/wikipedia.php"; + $special_request = new WikipediaRequest($query, $mh, $config); + break; + } + + return $special_request; + } +?> diff --git a/engines/special/currency.php b/engines/special/currency.php index 4c50849..fd11b79 100644 --- a/engines/special/currency.php +++ b/engines/special/currency.php @@ -1,7 +1,14 @@ ch); + + $split_query = explode(" ", $this->query); $base_currency = strtoupper($split_query[1]); $currency_to_convert = strtoupper($split_query[3]); diff --git a/engines/special/definition.php b/engines/special/definition.php index bd1f364..819eeb4 100644 --- a/engines/special/definition.php +++ b/engines/special/definition.php @@ -1,9 +1,14 @@ query); $reversed_split_q = array_reverse($split_query); $word_to_define = $reversed_split_q[1]; + return "https://api.dictionaryapi.dev/api/v2/entries/en/$word_to_define"; + } + + public function get_results() { + $response = curl_multi_getcontent($this->ch); $json_response = json_decode($response, true); @@ -20,5 +25,6 @@ ); } + } } ?> diff --git a/engines/special/ip.php b/engines/special/ip.php index 91a16f8..85a1abd 100644 --- a/engines/special/ip.php +++ b/engines/special/ip.php @@ -1,11 +1,17 @@ array( - "response" => $_SERVER["REMOTE_ADDR"], - "source" => null - ) - ); + class IPRequest extends EngineRequest { + function __construct($query, $mh, $config) { + $this->query = $query; + } + + function get_results() + { + return array( + "special_response" => array( + "response" => $_SERVER["REMOTE_ADDR"], + "source" => null + ) + ); + } } ?> diff --git a/engines/special/tor.php b/engines/special/tor.php index 451bf4e..d3172d0 100644 --- a/engines/special/tor.php +++ b/engines/special/tor.php @@ -1,18 +1,22 @@ array( "response" => $formatted_response, "source" => $source ) ); + } } ?> diff --git a/engines/special/user_agent.php b/engines/special/user_agent.php index 336606e..d557574 100644 --- a/engines/special/user_agent.php +++ b/engines/special/user_agent.php @@ -1,11 +1,17 @@ query = $query; + } + + function get_results() + { return array( "special_response" => array( "response" => $_SERVER["HTTP_USER_AGENT"], "source" => null ) ); + } } ?> diff --git a/engines/special/weather.php b/engines/special/weather.php index f98b158..2d1d0fb 100644 --- a/engines/special/weather.php +++ b/engines/special/weather.php @@ -1,26 +1,31 @@ ch); $json_response = json_decode($response, true); - if ($json_response) - { - $current_weather = $json_response["current_condition"][0]; + if (!$json_response) + return array(); - $temp_c = $current_weather["temp_C"]; - $temp_f = $current_weather["temp_F"]; - $description = $current_weather["weatherDesc"][0]["value"]; + $current_weather = $json_response["current_condition"][0]; - $formatted_response = "$description - $temp_c °C | $temp_f °F"; + $temp_c = $current_weather["temp_C"]; + $temp_f = $current_weather["temp_F"]; + $description = $current_weather["weatherDesc"][0]["value"]; - $source = "https://wttr.in"; - return array( - "special_response" => array( - "response" => htmlspecialchars($formatted_response), - "source" => $source - ) - ); - } + $formatted_response = "$description - $temp_c °C | $temp_f °F"; + $source = "https://wttr.in"; + return array( + "special_response" => array( + "response" => htmlspecialchars($formatted_response), + "source" => $source + ) + ); + } } ?> diff --git a/engines/special/wikipedia.php b/engines/special/wikipedia.php index bb8fd62..fdfd229 100644 --- a/engines/special/wikipedia.php +++ b/engines/special/wikipedia.php @@ -1,21 +1,27 @@ wikipedia_language = isset($_COOKIE["wikipedia_language"]) ? trim(htmlspecialchars($_COOKIE["wikipedia_language"])) : $this->config->wikipedia_language; + $query_encoded = urlencode($this->query); - $query_encoded = urlencode($query); + if (in_array($this->wikipedia_language, json_decode(file_get_contents("static/misc/wikipedia_langs.json"), true))) + return "https://$this->wikipedia_language.wikipedia.org/w/api.php?format=json&action=query&prop=extracts%7Cpageimages&exintro&explaintext&redirects=1&pithumbsize=500&titles=$query_encoded"; - $json_response = json_decode($response, true); + return ""; + } - $first_page = array_values($json_response["query"]["pages"])[0]; + public function get_results() { + $response = curl_multi_getcontent($this->ch); - if (!array_key_exists("missing", $first_page)) - { + $json_response = json_decode($response, true); + + $first_page = array_values($json_response["query"]["pages"])[0]; + + if (array_key_exists("missing", $first_page)) + return array(); $description = substr($first_page["extract"], 0, 250) . "..."; - $wikipedia_language = isset($_COOKIE["wikipedia_language"]) ? trim(htmlspecialchars($_COOKIE["wikipedia_language"])) : $config->wikipedia_language; - - $source = check_for_privacy_frontend("https://$wikipedia_language.wikipedia.org/wiki/$query"); + $source = check_for_privacy_frontend("https://$wikipedia_language.wikipedia.org/wiki/$this->query"); $response = array( "special_response" => array( "response" => htmlspecialchars($description), diff --git a/engines/text.php b/engines/text.php new file mode 100644 index 0000000..713602b --- /dev/null +++ b/engines/text.php @@ -0,0 +1,143 @@ +preferred_engines['text']; + if (is_null($engine)) + $engine = "google"; + $query_parts = explode(" ", $query); + $last_word_query = end($query_parts); + if (substr($query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!") + check_ddg_bang($query); + + if ($engine == "google") { + require "engines/google/text.php"; + $engine_request = new GoogleRequest($query, $mh, $config); + } + + if ($engine == "duckduckgo") { + require "engines/duckduckgo/text.php"; + $engine_request = new DuckDuckGoRequest($query, $mh, $config); + } + + require "engines/special.php"; + $special_request = get_special_search_request($query, $page, $mh, $config); + + $running = null; + + do { + curl_multi_exec($mh, $running); + } while ($running); + + if (curl_getinfo($)['http_code'] != '200') + { + require "engines/librex/text.php"; + return get_librex_results($query, $page); + } + + $results = $engine_request->get_results(); + + $special_result = $special_request->get_results(); + if ($special_result) + $results = array_merge(array($special_result), $results); + + return $results; + } + + function print_text_results($results) + { + + if (empty($results)) + return; + + $special = $results[0]; + + if (array_key_exists("did_you_mean", $special)) + { + $didyoumean = $special["did_you_mean"]; + $new_url = "/search.php?q=" . urlencode($didyoumean); + echo "

Did you mean "; + echo "$didyoumean"; + echo "?

"; + } + + if (array_key_exists("special_response", $special)) + { + $response = $special["special_response"]["response"]; + $source = $special["special_response"]["source"]; + + echo "

"; + if (array_key_exists("image", $special["special_response"])) + { + $image_url = $special["special_response"]["image"]; + echo ""; + } + echo $response; + if ($source) + echo "$source"; + echo "

"; + } + + echo "
"; + + foreach($results as $result) + { + if (!array_key_exists("title", $result)) + continue; + + $title = $result["title"]; + $url = $result["url"]; + $base_url = $result["base_url"]; + $description = $result["description"]; + + echo "
"; + echo ""; + echo "$base_url"; + echo "

$title

"; + echo ""; + echo "$description"; + echo "
"; + } + + echo "
"; + } + + function check_ddg_bang($query) + { + + $bangs_json = file_get_contents("static/misc/ddg_bang.json"); + $bangs = json_decode($bangs_json, true); + + if (substr($query, 0, 1) == "!") + $search_word = substr(explode(" ", $query)[0], 1); + else + $search_word = substr(end(explode(" ", $query)), 1); + + $bang_url = null; + + foreach($bangs as $bang) + { + if ($bang["t"] == $search_word) + { + $bang_url = $bang["u"]; + break; + } + } + + if ($bang_url) + { + $bang_query_array = explode("!" . $search_word, $query); + $bang_query = trim(implode("", $bang_query_array)); + + $request_url = str_replace("{{{s}}}", str_replace('%26quot%3B','%22', urlencode($bang_query)), $bang_url); + $request_url = check_for_privacy_frontend($request_url); + + header("Location: " . $request_url); + die(); + } + } + +?> diff --git a/misc/tools.php b/misc/tools.php index 9ea5f52..b70ba0e 100644 --- a/misc/tools.php +++ b/misc/tools.php @@ -101,86 +101,6 @@ return $url; } - function check_ddg_bang($query) - { - - $bangs_json = file_get_contents("static/misc/ddg_bang.json"); - $bangs = json_decode($bangs_json, true); - - if (substr($query, 0, 1) == "!") - $search_word = substr(explode(" ", $query)[0], 1); - else - $search_word = substr(end(explode(" ", $query)), 1); - - $bang_url = null; - - foreach($bangs as $bang) - { - if ($bang["t"] == $search_word) - { - $bang_url = $bang["u"]; - break; - } - } - - if ($bang_url) - { - $bang_query_array = explode("!" . $search_word, $query); - $bang_query = trim(implode("", $bang_query_array)); - - $request_url = str_replace("{{{s}}}", str_replace('%26quot%3B','%22', urlencode($bang_query)), $bang_url); - $request_url = check_for_privacy_frontend($request_url); - - header("Location: " . $request_url); - die(); - } - } - - function check_for_special_search($query) - { - if (isset($_COOKIE["disable_special"])) - return 0; - - $query_lower = strtolower($query); - $split_query = explode(" ", $query); - - if (strpos($query_lower, "to") && count($split_query) >= 4) // currency - { - $amount_to_convert = floatval($split_query[0]); - if ($amount_to_convert != 0) - return 1; - } - else if (strpos($query_lower, "mean") && count($split_query) >= 2) // definition - { - return 2; - } - else if (strpos($query_lower, "my") !== false) - { - if (strpos($query_lower, "ip")) - { - return 3; - } - else if (strpos($query_lower, "user agent") || strpos($query_lower, "ua")) - { - return 4; - } - } - else if (strpos($query_lower, "weather") !== false) - { - return 5; - } - else if ($query_lower == "tor") - { - return 6; - } - else if (3 > count(explode(" ", $query))) // wikipedia - { - return 7; - } - - return 0; - } - function get_xpath($response) { $htmlDom = new DOMDocument; diff --git a/search.php b/search.php index 27d45ec..e1bd8b6 100644 --- a/search.php +++ b/search.php @@ -3,11 +3,37 @@ $config = require "config.php"; require "misc/tools.php"; + $query = trim($_REQUEST["q"]); + + class EngineRequest { + public $ch; + public $query; + + function __construct($query, $mh, $config) { + $this->query = $query; + $this->config = $config; + + $url = $this->get_request_url(); + if ($url) { + $this->ch = curl_init($url); + curl_setopt_array($this->ch, $config->curl_settings); + curl_multi_add_handle($mh, $this->ch); + } + } + + public function get_request_url() { + return ""; + } + + public function get_results() { + return array(); + } + } + ?> <?php - $query = trim($_REQUEST["q"]); echo $query; ?> - LibreY @@ -56,20 +82,12 @@ preferred_engines['text']; - if (is_null($engine)) - $engine = "google"; - $query_parts = explode(" ", $query); - $last_word_query = end($query_parts); - if (substr($query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!") - check_ddg_bang($query); - require "engines/$engine/text.php"; + require "engines/text.php"; $results = get_text_results($query, $page); print_elapsed_time($start_time); print_text_results($results); From f59f94fdd5fb18bd2aae9893edd3f9e270da9191 Mon Sep 17 00:00:00 2001 From: davidovski Date: Mon, 21 Aug 2023 17:31:20 +0100 Subject: [PATCH 02/40] include different print and fetch functions for search --- api.php | 5 +- engines/ahmia/hidden_service.php | 3 +- engines/bittorrent/merge.php | 4 +- engines/duckduckgo/text.php | 8 +-- engines/invidious/video.php | 5 +- engines/qwant/image.php | 4 +- engines/text.php | 8 +-- search.php | 96 +++++++++++++------------------- 8 files changed, 54 insertions(+), 79 deletions(-) diff --git a/api.php b/api.php index 4413a48..3dcbe1a 100644 --- a/api.php +++ b/api.php @@ -26,10 +26,7 @@ switch ($type) { case 0: - $engine=$config->preferred_engines['text']; - if (is_null($engine)) - $engine = "google"; - require "engines/$engine/text.php"; + require "engines/text.php"; $results = get_text_results($query, $page); break; case 1: diff --git a/engines/ahmia/hidden_service.php b/engines/ahmia/hidden_service.php index d501b96..16a95db 100644 --- a/engines/ahmia/hidden_service.php +++ b/engines/ahmia/hidden_service.php @@ -1,5 +1,5 @@ diff --git a/engines/bittorrent/merge.php b/engines/bittorrent/merge.php index 1106ba3..9126ac9 100644 --- a/engines/bittorrent/merge.php +++ b/engines/bittorrent/merge.php @@ -1,6 +1,6 @@ "; diff --git a/engines/duckduckgo/text.php b/engines/duckduckgo/text.php index e4bdbac..0b4455b 100644 --- a/engines/duckduckgo/text.php +++ b/engines/duckduckgo/text.php @@ -6,11 +6,11 @@ $query_encoded = str_replace("%22", "\"", urlencode($this->query)); $results = array(); - // $domain = $config->google_domain; + // $domain = $this->config->google_domain; $domain = 'com'; - $site_language = isset($_COOKIE["google_language_site"]) ? trim(htmlspecialchars($_COOKIE["google_language_site"])) : $config->google_language_site; - $results_language = isset($_COOKIE["google_language_results"]) ? trim(htmlspecialchars($_COOKIE["google_language_results"])) : $config->google_language_results; - $number_of_results = isset($_COOKIE["google_number_of_results"]) ? trim(htmlspecialchars($_COOKIE["google_number_of_results"])) : $config->google_number_of_results; + $site_language = isset($_COOKIE["google_language_site"]) ? trim(htmlspecialchars($_COOKIE["google_language_site"])) : $this->config->google_language_site; + $results_language = isset($_COOKIE["google_language_results"]) ? trim(htmlspecialchars($_COOKIE["google_language_results"])) : $this->config->google_language_results; + $number_of_results = isset($_COOKIE["google_number_of_results"]) ? trim(htmlspecialchars($_COOKIE["google_number_of_results"])) : $this->config->google_number_of_results; $url = "https://html.duckduckgo.$domain/html/?q=$query_encoded&kd=-1&s=" . 3 * $page; if (3 > strlen($site_language) && 0 < strlen($site_language)) diff --git a/engines/invidious/video.php b/engines/invidious/video.php index 1d7076b..df32f5e 100644 --- a/engines/invidious/video.php +++ b/engines/invidious/video.php @@ -1,6 +1,5 @@ invidious_instance_for_video_results; @@ -38,7 +37,7 @@ return $results; } - function print_video_results($results) + function print_search_results($results) { echo "
"; diff --git a/engines/qwant/image.php b/engines/qwant/image.php index d5b7a16..9ecc8f4 100644 --- a/engines/qwant/image.php +++ b/engines/qwant/image.php @@ -1,5 +1,5 @@ "; diff --git a/engines/text.php b/engines/text.php index 713602b..b225316 100644 --- a/engines/text.php +++ b/engines/text.php @@ -1,6 +1,5 @@ ch)['http_code'] != '200') { require "engines/librex/text.php"; return get_librex_results($query, $page); @@ -47,8 +46,7 @@ return $results; } - function print_text_results($results) - { + function print_search_results($results) { if (empty($results)) return; diff --git a/search.php b/search.php index e1bd8b6..2f16369 100644 --- a/search.php +++ b/search.php @@ -30,6 +30,40 @@ } } + function import_search_category($type) { + switch ($type) + { + case 0: + require "engines/text.php"; + break; + + case 1: + require "engines/qwant/image.php"; + break; + + case 2: + require "engines/invidious/video.php"; + break; + + case 3: + if ($config->disable_bittorent_search) + echo "

The host disabled this feature! :C

"; + else + require "engines/bittorrent/merge.php"; + break; + + case 4: + if ($config->disable_hidden_service_search) + echo "

The host disabled this feature! :C

"; + else + require "engines/ahmia/hidden_service.php"; + break; + + default: + require "engines/text.php"; + break; + } + } ?> @@ -84,64 +118,10 @@ <?php $page = isset($_REQUEST["p"]) ? (int) $_REQUEST["p"] : 0; $start_time = microtime(true); - switch ($type) - { - case 0: - require "engines/text.php"; - $results = get_text_results($query, $page); - print_elapsed_time($start_time); - print_text_results($results); - break; - - case 1: - require "engines/qwant/image.php"; - $results = get_image_results($query_encoded, $page); - print_elapsed_time($start_time); - print_image_results($results); - break; - - case 2: - require "engines/invidious/video.php"; - $results = get_video_results($query_encoded); - print_elapsed_time($start_time); - print_video_results($results); - break; - - case 3: - if ($config->disable_bittorent_search) - echo "<p class=\"text-result-container\">The host disabled this feature! :C</p>"; - else - { - require "engines/bittorrent/merge.php"; - $results = get_merged_torrent_results($query_encoded); - print_elapsed_time($start_time); - print_merged_torrent_results($results); - } - break; - - case 4: - if ($config->disable_hidden_service_search) - echo "<p class=\"text-result-container\">The host disabled this feature! :C</p>"; - else - { - require "engines/ahmia/hidden_service.php"; - $results = get_hidden_service_results($query_encoded); - print_elapsed_time($start_time); - print_hidden_service_results($results); - } - break; - - default: - $query_parts = explode(" ", $query); - $last_word_query = end($query_parts); - if (substr($query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!") - check_ddg_bang($query); - require "engines/google/text.php"; - $results = get_text_results($query, $page); - print_elapsed_time($start_time); - print_text_results($results); - break; - } + import_search_category($type); + $results = fetch_search_results($query, $page); + print_elapsed_time($start_time); + print_search_results($results); if (2 > $type) From 74da63dab2b65e5ad1d458f2814eb1c5bc89ec3e Mon Sep 17 00:00:00 2001 From: davidovski <david@davidovski.xyz> Date: Mon, 21 Aug 2023 21:55:24 +0100 Subject: [PATCH 03/40] Make search categories type of request --- engines/duckduckgo/text.php | 35 ++++--- engines/google/text.php | 3 +- engines/invidious/video.php | 117 ++++++++++----------- engines/qwant/image.php | 100 +++++++++--------- engines/special.php | 15 +-- engines/special/ip.php | 2 +- engines/special/user_agent.php | 3 +- engines/special/wikipedia.php | 2 +- engines/text.php | 180 ++++++++++++++++----------------- search.php | 40 ++++---- 10 files changed, 250 insertions(+), 247 deletions(-) diff --git a/engines/duckduckgo/text.php b/engines/duckduckgo/text.php index 0b4455b..a781710 100644 --- a/engines/duckduckgo/text.php +++ b/engines/duckduckgo/text.php @@ -2,29 +2,28 @@ class DuckDuckGoRequest extends EngineRequest { function get_request_url() { - $page = isset($_REQUEST["p"]) ? (int) $_REQUEST["p"] : 0; - $query_encoded = str_replace("%22", "\"", urlencode($this->query)); - $results = array(); + $query_encoded = str_replace("%22", "\"", urlencode($this->query)); + $results = array(); - // $domain = $this->config->google_domain; - $domain = 'com'; - $site_language = isset($_COOKIE["google_language_site"]) ? trim(htmlspecialchars($_COOKIE["google_language_site"])) : $this->config->google_language_site; - $results_language = isset($_COOKIE["google_language_results"]) ? trim(htmlspecialchars($_COOKIE["google_language_results"])) : $this->config->google_language_results; - $number_of_results = isset($_COOKIE["google_number_of_results"]) ? trim(htmlspecialchars($_COOKIE["google_number_of_results"])) : $this->config->google_number_of_results; + // $domain = $this->config->google_domain; + $domain = 'com'; + $site_language = isset($_COOKIE["google_language_site"]) ? trim(htmlspecialchars($_COOKIE["google_language_site"])) : $this->config->google_language_site; + $results_language = isset($_COOKIE["google_language_results"]) ? trim(htmlspecialchars($_COOKIE["google_language_results"])) : $this->config->google_language_results; + $number_of_results = isset($_COOKIE["google_number_of_results"]) ? trim(htmlspecialchars($_COOKIE["google_number_of_results"])) : $this->config->google_number_of_results; - $url = "https://html.duckduckgo.$domain/html/?q=$query_encoded&kd=-1&s=" . 3 * $page; - if (3 > strlen($site_language) && 0 < strlen($site_language)) - $url .= "&hl=$site_language"; + $url = "https://html.duckduckgo.$domain/html/?q=$query_encoded&kd=-1&s=" . 3 * $this->page; + if (3 > strlen($site_language) && 0 < strlen($site_language)) + $url .= "&hl=$site_language"; - if (3 > strlen($results_language) && 0 < strlen($results_language)) - $url .= "&lr=lang_$results_language"; + if (3 > strlen($results_language) && 0 < strlen($results_language)) + $url .= "&lr=lang_$results_language"; - if (3 > strlen($number_of_results) && 0 < strlen($number_of_results)) - $url .= "&num=$number_of_results"; + if (3 > strlen($number_of_results) && 0 < strlen($number_of_results)) + $url .= "&num=$number_of_results"; - if (isset($_COOKIE["safe_search"])) - $url .= "&safe=medium"; - return $url; + if (isset($_COOKIE["safe_search"])) + $url .= "&safe=medium"; + return $url; } public function get_results() { diff --git a/engines/google/text.php b/engines/google/text.php index 323c0e7..eb2ac26 100644 --- a/engines/google/text.php +++ b/engines/google/text.php @@ -2,7 +2,6 @@ class GoogleRequest extends EngineRequest { function get_request_url() { - $page = isset($_REQUEST["p"]) ? (int) $_REQUEST["p"] : 0; $query_encoded = str_replace("%22", "\"", urlencode($this->query)); $results = array(); @@ -12,7 +11,7 @@ $results_language = isset($_COOKIE["google_language_results"]) ? trim(htmlspecialchars($_COOKIE["google_language_results"])) : $this->config->google_language_results; $number_of_results = isset($_COOKIE["google_number_of_results"]) ? trim(htmlspecialchars($_COOKIE["google_number_of_results"])) : $this->config->google_number_of_results; - $url = "https://www.google.$domain/search?q=$query_encoded&nfpr=1&start=$page"; + $url = "https://www.google.$domain/search?q=$query_encoded&nfpr=1&start=$this->page"; error_log($url); if (3 > strlen($site_language) && 0 < strlen($site_language)) diff --git a/engines/invidious/video.php b/engines/invidious/video.php index df32f5e..92dd6cf 100644 --- a/engines/invidious/video.php +++ b/engines/invidious/video.php @@ -1,67 +1,70 @@ <?php - function fetch_search_results($query, $page) { - global $config; - $instance_url = $config->invidious_instance_for_video_results; - - $url = "$instance_url/api/v1/search?q=$query"; - $response = request($url); - $json_response = json_decode($response, true); - $results = array(); - - foreach ($json_response as $response) - { - if ($response["type"] == "video") - { - $title = $response["title"]; - $url = "https://youtube.com/watch?v=" . $response["videoId"]; - $url = check_for_privacy_frontend($url); - $uploader = $response["author"]; - $views = $response["viewCount"]; - $date = $response["publishedText"]; - $thumbnail = $instance_url . "/vi/" . explode("/vi/" ,$response["videoThumbnails"][4]["url"])[1]; - - array_push($results, - array ( - "title" => htmlspecialchars($title), - "url" => htmlspecialchars($url), - "base_url" => htmlspecialchars(get_base_url($url)), - "uploader" => htmlspecialchars($uploader), - "views" => htmlspecialchars($views), - "date" => htmlspecialchars($date), - "thumbnail" => htmlspecialchars($thumbnail) - ) - ); - } + class VideoSearch extends EngineRequest { + public function get_request_url() { + $this->instance_url = $this->config->invidious_instance_for_video_results; + $query = urlencode($this->query); + return "$this->instance_url/api/v1/search?q=$query"; } - return $results; - } + public function get_results() { + $results = array(); + $response = curl_multi_getcontent($this->ch); + $json_response = json_decode($response, true); - function print_search_results($results) - { - echo "<div class=\"text-result-container\">"; - - foreach($results as $result) + foreach ($json_response as $response) { - $title = $result["title"]; - $url = $result["url"]; - $base_url = $result["base_url"]; - $uploader = $result["uploader"]; - $views = $result["views"]; - $date = $result["date"]; - $thumbnail = $result["thumbnail"]; + if ($response["type"] == "video") + { + $title = $response["title"]; + $url = "https://youtube.com/watch?v=" . $response["videoId"]; + $url = check_for_privacy_frontend($url); + $uploader = $response["author"]; + $views = $response["viewCount"]; + $date = $response["publishedText"]; + $thumbnail = $this->instance_url . "/vi/" . explode("/vi/" ,$response["videoThumbnails"][4]["url"])[1]; - echo "<div class=\"text-result-wrapper\">"; - echo "<a href=\"$url\">"; - echo "$base_url"; - echo "<h2>$title</h2>"; - echo "<img class=\"video-img\" src=\"image_proxy.php?url=$thumbnail\">"; - echo "<br>"; - echo "<span>$uploader - $date - $views views</span>"; - echo "</a>"; - echo "</div>"; + array_push($results, + array ( + "title" => htmlspecialchars($title), + "url" => htmlspecialchars($url), + "base_url" => htmlspecialchars(get_base_url($url)), + "uploader" => htmlspecialchars($uploader), + "views" => htmlspecialchars($views), + "date" => htmlspecialchars($date), + "thumbnail" => htmlspecialchars($thumbnail) + ) + ); + } } - echo "</div>"; + return $results; + } + + public function print_results($results) { + echo "<div class=\"text-result-container\">"; + + foreach($results as $result) + { + $title = $result["title"]; + $url = $result["url"]; + $base_url = $result["base_url"]; + $uploader = $result["uploader"]; + $views = $result["views"]; + $date = $result["date"]; + $thumbnail = $result["thumbnail"]; + + echo "<div class=\"text-result-wrapper\">"; + echo "<a href=\"$url\">"; + echo "$base_url"; + echo "<h2>$title</h2>"; + echo "<img class=\"video-img\" src=\"image_proxy.php?url=$thumbnail\">"; + echo "<br>"; + echo "<span>$uploader - $date - $views views</span>"; + echo "</a>"; + echo "</div>"; + } + + echo "</div>"; + } } ?> diff --git a/engines/qwant/image.php b/engines/qwant/image.php index 9ecc8f4..725db78 100644 --- a/engines/qwant/image.php +++ b/engines/qwant/image.php @@ -1,60 +1,60 @@ <?php - function fetch_search_results($query, $page) - { - global $config; - - $page = $page / 10 + 1; // qwant has a different page system - - $url = "https://lite.qwant.com/?q=$query&t=images&p=$page"; - $response = request($url); - $xpath = get_xpath($response); - - $results = array(); - - foreach($xpath->query("//a[@rel='noopener']") as $result) - { - $image = $xpath->evaluate(".//img", $result)[0]; - - if ($image) - { - $encoded_url = $result->getAttribute("href"); - $encoded_url_split1 = explode("==/", $encoded_url)[1]; - $encoded_url_split2 = explode("?position", $encoded_url_split1)[0]; - $real_url = urldecode(base64_decode($encoded_url_split2)); - $real_url = check_for_privacy_frontend($real_url); - - $alt = $image->getAttribute("alt"); - $thumbnail = urlencode($image->getAttribute("src")); - - array_push($results, - array ( - "thumbnail" => urldecode(htmlspecialchars($thumbnail)), - "alt" => htmlspecialchars($alt), - "url" => htmlspecialchars($real_url) - ) - ); - - } + class QwantImageSearch extends EngineRequest { + public function get_request_url() { + $page = $this->page / 10 + 1; // qwant has a different page system + $query = urlencode($this->query); + + return "https://lite.qwant.com/?q=$query&t=images&p=$page"; } - return $results; - } + public function get_results() { + $results = array(); + $xpath = get_xpath(curl_multi_getcontent($this->ch)); - function print_search_results($results) - { - echo "<div class=\"image-result-container\">"; + foreach($xpath->query("//a[@rel='noopener']") as $result) + { + $image = $xpath->evaluate(".//img", $result)[0]; - foreach($results as $result) - { - $thumbnail = urlencode($result["thumbnail"]); - $alt = $result["alt"]; - $url = $result["url"]; + if ($image) + { + $encoded_url = $result->getAttribute("href"); + $encoded_url_split1 = explode("==/", $encoded_url)[1]; + $encoded_url_split2 = explode("?position", $encoded_url_split1)[0]; + $real_url = urldecode(base64_decode($encoded_url_split2)); + $real_url = check_for_privacy_frontend($real_url); - echo "<a title=\"$alt\" href=\"$url\" target=\"_blank\">"; - echo "<img src=\"image_proxy.php?url=$thumbnail\">"; - echo "</a>"; + $alt = $image->getAttribute("alt"); + $thumbnail = urlencode($image->getAttribute("src")); + + array_push($results, + array ( + "thumbnail" => urldecode(htmlspecialchars($thumbnail)), + "alt" => htmlspecialchars($alt), + "url" => htmlspecialchars($real_url) + ) + ); + + } } - echo "</div>"; + return $results; + } + + public function print_results($results) { + echo "<div class=\"image-result-container\">"; + + foreach($results as $result) + { + $thumbnail = urlencode($result["thumbnail"]); + $alt = $result["alt"]; + $url = $result["url"]; + + echo "<a title=\"$alt\" href=\"$url\" target=\"_blank\">"; + echo "<img src=\"image_proxy.php?url=$thumbnail\">"; + echo "</a>"; + } + + echo "</div>"; + } } ?> diff --git a/engines/special.php b/engines/special.php index e453057..15a4700 100644 --- a/engines/special.php +++ b/engines/special.php @@ -1,5 +1,6 @@ <?php + // TODO make this a EngineRequest also function check_for_special_search($query) { if (isset($_COOKIE["disable_special"])) return 0; @@ -55,31 +56,31 @@ switch ($special_search) { case 1: require "engines/special/currency.php"; - $special_request = new CurrencyRequest($query, $mh, $config); + $special_request = new CurrencyRequest($query, $page, $mh, $config); break; case 2: require "engines/special/definition.php"; - $special_request = new DefinitionRequest($query, $mh, $config); + $special_request = new DefinitionRequest($query, $page, $mh, $config); break; case 3: require "engines/special/ip.php"; - $special_request = new IPRequest($query, $mh, $config); + $special_request = new IPRequest($query, $page, $mh, $config); break; case 4: require "engines/special/user_agent.php"; - $special_request = new UserAgentRequest($query, $mh, $config); + $special_request = new UserAgentRequest($query, $page, $mh, $config); break; case 5: require "engines/special/weather.php"; - $special_request = new WeatherRequest($query, $mh, $config); + $special_request = new WeatherRequest($query, $page, $mh, $config); break; case 6: require "engines/special/tor.php"; - $special_request = new TorRequest($query, $mh, $config); + $special_request = new TorRequest($query, $page, $mh, $config); break; case 7: require "engines/special/wikipedia.php"; - $special_request = new WikipediaRequest($query, $mh, $config); + $special_request = new WikipediaRequest($query, $page, $mh, $config); break; } diff --git a/engines/special/ip.php b/engines/special/ip.php index 85a1abd..1e7cab9 100644 --- a/engines/special/ip.php +++ b/engines/special/ip.php @@ -1,6 +1,6 @@ <?php class IPRequest extends EngineRequest { - function __construct($query, $mh, $config) { + function __construct($query, $page, $mh, $config) { $this->query = $query; } diff --git a/engines/special/user_agent.php b/engines/special/user_agent.php index d557574..aab3d59 100644 --- a/engines/special/user_agent.php +++ b/engines/special/user_agent.php @@ -1,7 +1,8 @@ <?php class UserAgentRequest extends EngineRequest { - function __construct($query, $mh, $config) { + function __construct($query, $page, $mh, $config) { $this->query = $query; + $this->page = $page; } function get_results() diff --git a/engines/special/wikipedia.php b/engines/special/wikipedia.php index fdfd229..8f5189e 100644 --- a/engines/special/wikipedia.php +++ b/engines/special/wikipedia.php @@ -21,7 +21,7 @@ return array(); $description = substr($first_page["extract"], 0, 250) . "..."; - $source = check_for_privacy_frontend("https://$wikipedia_language.wikipedia.org/wiki/$this->query"); + $source = check_for_privacy_frontend("https://$this->wikipedia_language.wikipedia.org/wiki/$this->query"); $response = array( "special_response" => array( "response" => htmlspecialchars($description), diff --git a/engines/text.php b/engines/text.php index b225316..f9d78d3 100644 --- a/engines/text.php +++ b/engines/text.php @@ -1,106 +1,100 @@ <?php - function fetch_search_results($query, $page) { - global $config; + class TextSearch extends EngineRequest { + public function __construct($query, $page, $mh, $config) { + $engine=$config->preferred_engines['text']; + if (is_null($engine)) + $engine = "google"; + $query_parts = explode(" ", $query); + $last_word_query = end($query_parts); + if (substr($query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!") + check_ddg_bang($query); - $mh = curl_multi_init(); - - $engine=$config->preferred_engines['text']; - if (is_null($engine)) - $engine = "google"; - $query_parts = explode(" ", $query); - $last_word_query = end($query_parts); - if (substr($query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!") - check_ddg_bang($query); - - if ($engine == "google") { - require "engines/google/text.php"; - $engine_request = new GoogleRequest($query, $mh, $config); - } - - if ($engine == "duckduckgo") { - require "engines/duckduckgo/text.php"; - $engine_request = new DuckDuckGoRequest($query, $mh, $config); - } - - require "engines/special.php"; - $special_request = get_special_search_request($query, $page, $mh, $config); - - $running = null; - - do { - curl_multi_exec($mh, $running); - } while ($running); - - if (curl_getinfo($engine_request->ch)['http_code'] != '200') - { - require "engines/librex/text.php"; - return get_librex_results($query, $page); - } - - $results = $engine_request->get_results(); - - $special_result = $special_request->get_results(); - if ($special_result) - $results = array_merge(array($special_result), $results); - - return $results; - } - - function print_search_results($results) { - - if (empty($results)) - return; - - $special = $results[0]; - - if (array_key_exists("did_you_mean", $special)) - { - $didyoumean = $special["did_you_mean"]; - $new_url = "/search.php?q=" . urlencode($didyoumean); - echo "<p class=\"did-you-mean\">Did you mean "; - echo "<a href=\"$new_url\">$didyoumean</a>"; - echo "?</p>"; - } - - if (array_key_exists("special_response", $special)) - { - $response = $special["special_response"]["response"]; - $source = $special["special_response"]["source"]; - - echo "<p class=\"special-result-container\">"; - if (array_key_exists("image", $special["special_response"])) - { - $image_url = $special["special_response"]["image"]; - echo "<img src=\"image_proxy.php?url=$image_url\">"; + if ($engine == "google") { + require "engines/google/text.php"; + $this->engine_request = new GoogleRequest($query, $page, $mh, $config); } - echo $response; - if ($source) - echo "<a href=\"$source\" target=\"_blank\">$source</a>"; - echo "</p>"; + + if ($engine == "duckduckgo") { + require "engines/duckduckgo/text.php"; + $this->engine_request = new DuckDuckGoRequest($query, $page, $mh, $config); + } + + require "engines/special.php"; + $this->special_request = get_special_search_request($query, $page, $mh, $config); } - echo "<div class=\"text-result-container\">"; + public function get_results() { + if (curl_getinfo($this->engine_request->ch)['http_code'] != '200') + { + require "engines/librex/text.php"; + return get_librex_results($this->query, $this->page); + } - foreach($results as $result) - { - if (!array_key_exists("title", $result)) - continue; + $results = $this->engine_request->get_results(); - $title = $result["title"]; - $url = $result["url"]; - $base_url = $result["base_url"]; - $description = $result["description"]; + $special_result = $this->special_request->get_results(); + if ($special_result) + $results = array_merge(array($special_result), $results); + + return $results; + } + + public function print_results($results) { + + if (empty($results)) + return; + + $special = $results[0]; + + if (array_key_exists("did_you_mean", $special)) + { + $didyoumean = $special["did_you_mean"]; + $new_url = "/search.php?q=" . urlencode($didyoumean); + echo "<p class=\"did-you-mean\">Did you mean "; + echo "<a href=\"$new_url\">$didyoumean</a>"; + echo "?</p>"; + } + + if (array_key_exists("special_response", $special)) + { + $response = $special["special_response"]["response"]; + $source = $special["special_response"]["source"]; + + echo "<p class=\"special-result-container\">"; + if (array_key_exists("image", $special["special_response"])) + { + $image_url = $special["special_response"]["image"]; + echo "<img src=\"image_proxy.php?url=$image_url\">"; + } + echo $response; + if ($source) + echo "<a href=\"$source\" target=\"_blank\">$source</a>"; + echo "</p>"; + } + + echo "<div class=\"text-result-container\">"; + + foreach($results as $result) + { + if (!array_key_exists("title", $result)) + continue; + + $title = $result["title"]; + $url = $result["url"]; + $base_url = $result["base_url"]; + $description = $result["description"]; + + echo "<div class=\"text-result-wrapper\">"; + echo "<a href=\"$url\">"; + echo "$base_url"; + echo "<h2>$title</h2>"; + echo "</a>"; + echo "<span>$description</span>"; + echo "</div>"; + } - echo "<div class=\"text-result-wrapper\">"; - echo "<a href=\"$url\">"; - echo "$base_url"; - echo "<h2>$title</h2>"; - echo "</a>"; - echo "<span>$description</span>"; echo "</div>"; } - - echo "</div>"; } function check_ddg_bang($query) diff --git a/search.php b/search.php index 2f16369..b061c5c 100644 --- a/search.php +++ b/search.php @@ -5,44 +5,42 @@ require "misc/tools.php"; $query = trim($_REQUEST["q"]); - class EngineRequest { - public $ch; - public $query; - - function __construct($query, $mh, $config) { + abstract class EngineRequest { + function __construct($query, $page, $mh, $config) { $this->query = $query; + $this->page = $page; $this->config = $config; $url = $this->get_request_url(); if ($url) { + error_log($url); $this->ch = curl_init($url); curl_setopt_array($this->ch, $config->curl_settings); curl_multi_add_handle($mh, $this->ch); } } - public function get_request_url() { + abstract function get_results(); + public function get_request_url(){ return ""; } - - public function get_results() { - return array(); - } + public function print_results($results){} } - function import_search_category($type) { + function init_search($type, $query, $page, $mh, $config) { switch ($type) { case 0: require "engines/text.php"; - break; + return new TextSearch($query, $page, $mh, $config); case 1: require "engines/qwant/image.php"; - break; + return new QwantImageSearch($query, $page, $mh, $config); case 2: require "engines/invidious/video.php"; + return new VideoSearch($query, $page, $mh, $config); break; case 3: @@ -118,11 +116,19 @@ <?php $page = isset($_REQUEST["p"]) ? (int) $_REQUEST["p"] : 0; $start_time = microtime(true); - import_search_category($type); - $results = fetch_search_results($query, $page); - print_elapsed_time($start_time); - print_search_results($results); + $mh = curl_multi_init(); + $search_category = init_search($type, $query, $page, $mh, $config); + + $running = null; + + do { + curl_multi_exec($mh, $running); + } while ($running); + + $results = $search_category->get_results($query, $page); + print_elapsed_time($start_time); + $search_category->print_results($results); if (2 > $type) { From 19627c94c9575160c68429d9b5ceef7780b3d33a Mon Sep 17 00:00:00 2001 From: davidovski <david@davidovski.xyz> Date: Mon, 21 Aug 2023 23:28:02 +0100 Subject: [PATCH 04/40] add bittorrent and tor to be enginerequest types --- engines/ahmia/hidden_service.php | 58 +++++----- engines/bittorrent/merge.php | 187 +++++++++++++++---------------- engines/invidious/video.php | 2 +- engines/qwant/image.php | 2 +- engines/text.php | 5 +- misc/engine_request.php | 58 ++++++++++ search.php | 65 +---------- 7 files changed, 193 insertions(+), 184 deletions(-) create mode 100644 misc/engine_request.php diff --git a/engines/ahmia/hidden_service.php b/engines/ahmia/hidden_service.php index 16a95db..e88da58 100644 --- a/engines/ahmia/hidden_service.php +++ b/engines/ahmia/hidden_service.php @@ -1,31 +1,39 @@ <?php - function get_search_results($query) - { - global $config; + require "engines/text.php"; - $url = "https://ahmia.fi/search/?q=$query"; - $response = request($url); - $xpath = get_xpath($response); - - $results = array(); - - foreach($xpath->query("//ol[@class='searchResults']//li[@class='result']") as $result) - { - $url = "http://" . $xpath->evaluate(".//cite", $result)[0]->textContent; - $title = remove_special($xpath->evaluate(".//h4", $result)[0]->textContent); - $description = $xpath->evaluate(".//p", $result)[0]->textContent; - - array_push($results, - array ( - "title" => $title ? htmlspecialchars($title) : "No description provided", - "url" => htmlspecialchars($url), - "base_url" => htmlspecialchars(get_base_url($url)), - "description" => htmlspecialchars($description) - ) - ); + class TorSearch extends EngineRequest { + public function get_request_url() { + $query = urlencode($this->query); + return "https://ahmia.fi/search/?q=$query"; } - return $results; + public function get_results() { + $response = curl_multi_getcontent($this->ch); + $xpath = get_xpath($response); + + $results = array(); + + foreach($xpath->query("//ol[@class='searchResults']//li[@class='result']") as $result) + { + $url = "http://" . $xpath->evaluate(".//cite", $result)[0]->textContent; + $title = remove_special($xpath->evaluate(".//h4", $result)[0]->textContent); + $description = $xpath->evaluate(".//p", $result)[0]->textContent; + + array_push($results, + array ( + "title" => $title ? htmlspecialchars($title) : "No description provided", + "url" => htmlspecialchars($url), + "base_url" => htmlspecialchars(get_base_url($url)), + "description" => htmlspecialchars($description) + ) + ); + } + + return $results; + } + + public static function print_results($results) { + TextSearch::print_results($results); + } } - require "engines/text.php"; ?> diff --git a/engines/bittorrent/merge.php b/engines/bittorrent/merge.php index 9126ac9..be4727f 100644 --- a/engines/bittorrent/merge.php +++ b/engines/bittorrent/merge.php @@ -1,111 +1,110 @@ <?php + class TorrentSearch extends EngineRequest { + public function __construct($query, $page, $mh, $config) { + $this->query = $query; + $this->page = $page; - function fetch_search_results($query) - { - global $config; + // TODO make these engine requests + require "engines/bittorrent/thepiratebay.php"; + require "engines/bittorrent/rutor.php"; + require "engines/bittorrent/nyaa.php"; + require "engines/bittorrent/yts.php"; + require "engines/bittorrent/torrentgalaxy.php"; + require "engines/bittorrent/1337x.php"; + require "engines/bittorrent/sukebei.php"; - require "engines/bittorrent/thepiratebay.php"; - require "engines/bittorrent/rutor.php"; - require "engines/bittorrent/nyaa.php"; - require "engines/bittorrent/yts.php"; - require "engines/bittorrent/torrentgalaxy.php"; - require "engines/bittorrent/1337x.php"; - require "engines/bittorrent/sukebei.php"; + $query = urlencode($query); - $query = urlencode($query); + $torrent_urls = array( + $thepiratebay_url, + $rutor_url, + $nyaa_url, + $yts_url, + $torrentgalaxy_url, + $_1337x_url, + $sukebei_url + ); - $torrent_urls = array( - $thepiratebay_url, - $rutor_url, - $nyaa_url, - $yts_url, - $torrentgalaxy_url, - $_1337x_url, - $sukebei_url - ); + $this->chs = array(); - $mh = curl_multi_init(); - $chs = $results = array(); - - foreach ($torrent_urls as $url) - { - $ch = curl_init($url); - curl_setopt_array($ch, $config->curl_settings); - array_push($chs, $ch); - curl_multi_add_handle($mh, $ch); - } - - $running = null; - do { - curl_multi_exec($mh, $running); - } while ($running); - - for ($i=0; count($chs)>$i; $i++) - { - $response = curl_multi_getcontent($chs[$i]); - - switch ($i) + foreach ($torrent_urls as $url) { - case 0: - $results = array_merge($results, get_thepiratebay_results($response)); - break; - case 1: - $results = array_merge($results, get_rutor_results($response)); - break; - case 2: - $results = array_merge($results, get_nyaa_results($response)); - break; - case 3: - $results = array_merge($results, get_yts_results($response)); - break; - case 4: - $results = array_merge($results, get_torrentgalaxy_results($response)); - break; - case 5: - $results = array_merge($results, get_1337x_results($response)); - break; - case 6: - $results = array_merge($results, get_sukebei_results($response)); - break; + $ch = curl_init($url); + curl_setopt_array($ch, $config->curl_settings); + array_push($this->chs, $ch); + curl_multi_add_handle($mh, $ch); } } - $seeders = array_column($results, "seeders"); - array_multisort($seeders, SORT_DESC, $results); - - return $results; - } - - function print_search_results($results) - { - echo "<div class=\"text-result-container\">"; - - if (!empty($results)) - { - foreach($results as $result) + public function get_results() { + $query = urlencode($this->query); + $results = array(); + for ($i=0; count($this->chs)>$i; $i++) { - $source = $result["source"]; - $name = $result["name"]; - $magnet = $result["magnet"]; - $seeders = $result["seeders"]; - $leechers = $result["leechers"]; - $size = $result["size"]; + $response = curl_multi_getcontent($this->chs[$i]); - echo "<div class=\"text-result-wrapper\">"; - echo "<a href=\"$magnet\">"; - echo "$source"; - echo "<h2>$name</h2>"; - echo "</a>"; - echo "<span>SE: <span class=\"seeders\">$seeders</span> - "; - echo "LE: <span class=\"leechers\">$leechers</span> - "; - echo "$size</span>"; - echo "</div>"; + switch ($i) + { + case 0: + $results = array_merge($results, get_thepiratebay_results($response)); + break; + case 1: + $results = array_merge($results, get_rutor_results($response)); + break; + case 2: + $results = array_merge($results, get_nyaa_results($response)); + break; + case 3: + $results = array_merge($results, get_yts_results($response)); + break; + case 4: + //$results = array_merge($results, get_torrentgalaxy_results($response)); + break; + case 5: + //$results = array_merge($results, get_1337x_results($response)); + break; + case 6: + $results = array_merge($results, get_sukebei_results($response)); + break; + } } - } - else - echo "<p>There are no results. Please try different keywords!</p>"; - echo "</div>"; + $seeders = array_column($results, "seeders"); + array_multisort($seeders, SORT_DESC, $results); + + return $results; + } + + public static function print_results($results) { + echo "<div class=\"text-result-container\">"; + + if (!empty($results)) + { + foreach($results as $result) + { + $source = $result["source"]; + $name = $result["name"]; + $magnet = $result["magnet"]; + $seeders = $result["seeders"]; + $leechers = $result["leechers"]; + $size = $result["size"]; + + echo "<div class=\"text-result-wrapper\">"; + echo "<a href=\"$magnet\">"; + echo "$source"; + echo "<h2>$name</h2>"; + echo "</a>"; + echo "<span>SE: <span class=\"seeders\">$seeders</span> - "; + echo "LE: <span class=\"leechers\">$leechers</span> - "; + echo "$size</span>"; + echo "</div>"; + } + } + else + echo "<p>There are no results. Please try different keywords!</p>"; + + echo "</div>"; + } } ?> diff --git a/engines/invidious/video.php b/engines/invidious/video.php index 92dd6cf..7fefbf6 100644 --- a/engines/invidious/video.php +++ b/engines/invidious/video.php @@ -40,7 +40,7 @@ return $results; } - public function print_results($results) { + public static function print_results($results) { echo "<div class=\"text-result-container\">"; foreach($results as $result) diff --git a/engines/qwant/image.php b/engines/qwant/image.php index 725db78..a32dfce 100644 --- a/engines/qwant/image.php +++ b/engines/qwant/image.php @@ -40,7 +40,7 @@ return $results; } - public function print_results($results) { + public static function print_results($results) { echo "<div class=\"image-result-container\">"; foreach($results as $result) diff --git a/engines/text.php b/engines/text.php index f9d78d3..9506318 100644 --- a/engines/text.php +++ b/engines/text.php @@ -1,6 +1,9 @@ <?php class TextSearch extends EngineRequest { public function __construct($query, $page, $mh, $config) { + $this->query = $query; + $this->page = $page; + $engine=$config->preferred_engines['text']; if (is_null($engine)) $engine = "google"; @@ -39,7 +42,7 @@ return $results; } - public function print_results($results) { + public static function print_results($results) { if (empty($results)) return; diff --git a/misc/engine_request.php b/misc/engine_request.php new file mode 100644 index 0000000..9a47bf3 --- /dev/null +++ b/misc/engine_request.php @@ -0,0 +1,58 @@ +<?php + abstract class EngineRequest { + function __construct($query, $page, $mh, $config) { + $this->query = $query; + $this->page = $page; + $this->config = $config; + + $url = $this->get_request_url(); + if ($url) { + error_log($url); + $this->ch = curl_init($url); + curl_setopt_array($this->ch, $config->curl_settings); + curl_multi_add_handle($mh, $this->ch); + } + } + + public function get_request_url(){ + return ""; + } + + abstract function get_results(); + static public function print_results($results){} + } + + function init_search($type, $query, $page, $mh, $config) { + switch ($type) + { + case 1: + require "engines/qwant/image.php"; + return new QwantImageSearch($query, $page, $mh, $config); + + case 2: + require "engines/invidious/video.php"; + return new VideoSearch($query, $page, $mh, $config); + + case 3: + if ($config->disable_bittorent_search) { + echo "<p class=\"text-result-container\">The host disabled this feature! :C</p>"; + break; + } + + require "engines/bittorrent/merge.php"; + return new TorrentSearch($query, $page, $mh, $config); + + case 4: + if ($config->disable_hidden_service_search) { + echo "<p class=\"text-result-container\">The host disabled this feature! :C</p>"; + break; + } + require "engines/ahmia/hidden_service.php"; + return new TorSearch($query, $page, $mh, $config); + + default: + require "engines/text.php"; + return new TextSearch($query, $page, $mh, $config); + } + } +?> diff --git a/search.php b/search.php index b061c5c..baf454f 100644 --- a/search.php +++ b/search.php @@ -3,70 +3,13 @@ $config = require "config.php"; require "misc/tools.php"; - $query = trim($_REQUEST["q"]); - - abstract class EngineRequest { - function __construct($query, $page, $mh, $config) { - $this->query = $query; - $this->page = $page; - $this->config = $config; - - $url = $this->get_request_url(); - if ($url) { - error_log($url); - $this->ch = curl_init($url); - curl_setopt_array($this->ch, $config->curl_settings); - curl_multi_add_handle($mh, $this->ch); - } - } - - abstract function get_results(); - public function get_request_url(){ - return ""; - } - public function print_results($results){} - } - - function init_search($type, $query, $page, $mh, $config) { - switch ($type) - { - case 0: - require "engines/text.php"; - return new TextSearch($query, $page, $mh, $config); - - case 1: - require "engines/qwant/image.php"; - return new QwantImageSearch($query, $page, $mh, $config); - - case 2: - require "engines/invidious/video.php"; - return new VideoSearch($query, $page, $mh, $config); - break; - - case 3: - if ($config->disable_bittorent_search) - echo "<p class=\"text-result-container\">The host disabled this feature! :C</p>"; - else - require "engines/bittorrent/merge.php"; - break; - - case 4: - if ($config->disable_hidden_service_search) - echo "<p class=\"text-result-container\">The host disabled this feature! :C</p>"; - else - require "engines/ahmia/hidden_service.php"; - break; - - default: - require "engines/text.php"; - break; - } - } + require "misc/engine_request.php"; ?> <title> <?php - echo $query; + $query = trim($_REQUEST["q"]); + echo $query; ?> - LibreY @@ -74,8 +17,6 @@

LibreY

strlen($query) || strlen($query) > 256) { header("Location: ./"); From 34fdaac05826f574731c91a787039c9eb4326998 Mon Sep 17 00:00:00 2001 From: davidovski Date: Mon, 21 Aug 2023 23:58:09 +0100 Subject: [PATCH 05/40] use same logic for search and api --- api.php | 42 +------------ .../{engine_request.php => search_engine.php} | 23 ++++++- search.php | 62 ++++++++----------- 3 files changed, 50 insertions(+), 77 deletions(-) rename misc/{engine_request.php => search_engine.php} (76%) diff --git a/api.php b/api.php index 3dcbe1a..a18046e 100644 --- a/api.php +++ b/api.php @@ -1,6 +1,7 @@ disable_bittorent_search) - $results = array("error" => "disabled"); - else - { - require "engines/bittorrent/merge.php"; - $results = get_merged_torrent_results($query_encoded); - } - break; - case 4: - if ($config->disable_hidden_service_search) - $results = array("error" => "disabled"); - else - { - require "engines/ahmia/hidden_service.php"; - $results = get_hidden_service_results($query_encoded); - } - break; - default: - require "engines/google/text.php"; - $results = get_text_results($query_encoded, $page); - break; - } - + $results = fetch_search_results($type, $query, $page, $config, false); header("Content-Type: application/json"); echo json_encode($results); ?> diff --git a/misc/engine_request.php b/misc/search_engine.php similarity index 76% rename from misc/engine_request.php rename to misc/search_engine.php index 9a47bf3..39e4a64 100644 --- a/misc/engine_request.php +++ b/misc/search_engine.php @@ -7,7 +7,6 @@ $url = $this->get_request_url(); if ($url) { - error_log($url); $this->ch = curl_init($url); curl_setopt_array($this->ch, $config->curl_settings); curl_multi_add_handle($mh, $this->ch); @@ -55,4 +54,26 @@ return new TextSearch($query, $page, $mh, $config); } } + + function fetch_search_results($type, $query, $page, $config, $do_print) { + $start_time = microtime(true); + $mh = curl_multi_init(); + $search_category = init_search($type, $query, $page, $mh, $config); + + $running = null; + + do { + curl_multi_exec($mh, $running); + } while ($running); + + $results = $search_category->get_results($query, $page); + + if (!$do_print) + return $results; + + print_elapsed_time($start_time); + $search_category->print_results($results); + + return $results; + } ?> diff --git a/search.php b/search.php index baf454f..4ebd3ee 100644 --- a/search.php +++ b/search.php @@ -3,12 +3,34 @@ $config = require "config.php"; require "misc/tools.php"; - require "misc/engine_request.php"; + require "misc/search_engine.php"; + + function print_page_buttons($type, $query, $page) { + if ($type > 1) + return; + echo "
"; + + if ($page != 0) + { + print_next_page_button("<<", 0, $query, $type); + print_next_page_button("<", $page - 10, $query, $type); + } + + for ($i=$page / 10; $page / 10 + 10 > $i; $i++) + print_next_page_button($i + 1, $i * 10, $query, $type); + + print_next_page_button(">", $page + 10, $query, $type); + + echo "
"; + } + + $query = trim($_REQUEST["q"]); + $type = isset($_REQUEST["t"]) ? (int) $_REQUEST["t"] : 0; + $page = isset($_REQUEST["p"]) ? (int) $_REQUEST["p"] : 0; ?> <?php - $query = trim($_REQUEST["q"]); echo $query; ?> - LibreY @@ -28,7 +50,6 @@ >
"; ?> @@ -55,39 +76,8 @@ get_results($query, $page); - print_elapsed_time($start_time); - $search_category->print_results($results); - - if (2 > $type) - { - echo "
"; - - if ($page != 0) - { - print_next_page_button("<<", 0, $query, $type); - print_next_page_button("<", $page - 10, $query, $type); - } - - for ($i=$page / 10; $page / 10 + 10 > $i; $i++) - print_next_page_button($i + 1, $i * 10, $query, $type); - - print_next_page_button(">", $page + 10, $query, $type); - - echo "
"; - } + fetch_search_results($type, $query, $page, $config, true); + print_page_buttons($type, $query, $page); ?> From f4a80135bd2888e73376a52d9236beacd6455ff0 Mon Sep 17 00:00:00 2001 From: davidovski Date: Tue, 22 Aug 2023 00:05:35 +0100 Subject: [PATCH 06/40] ensure url remaisn the same when switching types --- search.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search.php b/search.php index 4ebd3ee..07319d3 100644 --- a/search.php +++ b/search.php @@ -68,7 +68,7 @@ continue; } - echo "\""" . ucfirst($category) . ""; + echo "\""" . ucfirst($category) . ""; } ?>
From 0bd726d897a48c6b4a281bded498148be694bfcc Mon Sep 17 00:00:00 2001 From: davidovski Date: Tue, 22 Aug 2023 00:16:22 +0100 Subject: [PATCH 07/40] moved files --- engines/{ => special}/special.php | 1 - .../{duckduckgo/text.php => text/duckduckgo.php} | 0 engines/{google/text.php => text/google.php} | 0 engines/{ => text}/text.php | 15 +++++++++------ misc/search_engine.php | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) rename engines/{ => special}/special.php (98%) rename engines/{duckduckgo/text.php => text/duckduckgo.php} (100%) rename engines/{google/text.php => text/google.php} (100%) rename engines/{ => text}/text.php (91%) diff --git a/engines/special.php b/engines/special/special.php similarity index 98% rename from engines/special.php rename to engines/special/special.php index 15a4700..d149ee2 100644 --- a/engines/special.php +++ b/engines/special/special.php @@ -1,6 +1,5 @@ engine_request = new GoogleRequest($query, $page, $mh, $config); } if ($engine == "duckduckgo") { - require "engines/duckduckgo/text.php"; + require "engines/text/duckduckgo.php"; $this->engine_request = new DuckDuckGoRequest($query, $page, $mh, $config); } - require "engines/special.php"; + require "engines/special/special.php"; $this->special_request = get_special_search_request($query, $page, $mh, $config); } @@ -35,9 +35,12 @@ $results = $this->engine_request->get_results(); - $special_result = $this->special_request->get_results(); - if ($special_result) - $results = array_merge(array($special_result), $results); + if ($this->special_request) { + $special_result = $this->special_request->get_results(); + + if ($special_result) + $results = array_merge(array($special_result), $results); + } return $results; } diff --git a/misc/search_engine.php b/misc/search_engine.php index 39e4a64..28a8b26 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -50,7 +50,7 @@ return new TorSearch($query, $page, $mh, $config); default: - require "engines/text.php"; + require "engines/text/text.php"; return new TextSearch($query, $page, $mh, $config); } } From 4941a6aa58aeb96a95fc02c7a836a115e6c4dbad Mon Sep 17 00:00:00 2001 From: davidovski Date: Tue, 22 Aug 2023 13:25:44 +0100 Subject: [PATCH 08/40] use engine request for torrent search --- engines/bittorrent/1337x.php | 63 ++++++++-------- engines/bittorrent/merge.php | 107 +++++++++------------------ engines/bittorrent/nyaa.php | 73 ++++++++++-------- engines/bittorrent/rutor.php | 65 ++++++++-------- engines/bittorrent/sukebei.php | 44 +---------- engines/bittorrent/thepiratebay.php | 74 +++++++++--------- engines/bittorrent/torrentgalaxy.php | 64 ++++++++-------- engines/bittorrent/yts.php | 82 ++++++++++---------- misc/search_engine.php | 4 + 9 files changed, 268 insertions(+), 308 deletions(-) diff --git a/engines/bittorrent/1337x.php b/engines/bittorrent/1337x.php index 21a80eb..e7ae3e5 100644 --- a/engines/bittorrent/1337x.php +++ b/engines/bittorrent/1337x.php @@ -1,34 +1,39 @@ query("//table/tbody/tr") as $result) - { - - $name = $xpath->evaluate(".//td[@class='coll-1 name']/a", $result)[1]->textContent; - $magnet = "./engines/bittorrent/get_magnet_1337x.php?url=https://1337x.to" . $xpath->evaluate(".//td[@class='coll-1 name']/a/@href", $result)[1]->textContent; - $size_unformatted = explode(" ", $xpath->evaluate(".//td[contains(@class, 'coll-4 size')]", $result)[0]->textContent); - $size = $size_unformatted[0] . " " . preg_replace("/[0-9]+/", "", $size_unformatted[1]); - $seeders = $xpath->evaluate(".//td[@class='coll-2 seeds']", $result)[0]->textContent; - $leechers = $xpath->evaluate(".//td[@class='coll-3 leeches']", $result)[0]->textContent; - - array_push($results, - array ( - "name" => htmlspecialchars($name), - "seeders" => (int) $seeders, - "leechers" => (int) $leechers, - "magnet" => htmlspecialchars($magnet), - "size" => htmlspecialchars($size), - "source" => "1337x.to" - ) - ); + class _1337xRequest extends EngineRequest { + public function get_request_url() { + $query = urlencode($this->query); + return "https://1337x.to/search/$query/1/"; } - return $results; + public function get_results() { + $response = curl_multi_getcontent($this->ch); + + $xpath = get_xpath($response); + $results = array(); + + foreach($xpath->query("//table/tbody/tr") as $result) + { + + $name = $xpath->evaluate(".//td[@class='coll-1 name']/a", $result)[1]->textContent; + $magnet = "./engines/bittorrent/get_magnet_1337x.php?url=https://1337x.to" . $xpath->evaluate(".//td[@class='coll-1 name']/a/@href", $result)[1]->textContent; + $size_unformatted = explode(" ", $xpath->evaluate(".//td[contains(@class, 'coll-4 size')]", $result)[0]->textContent); + $size = $size_unformatted[0] . " " . preg_replace("/[0-9]+/", "", $size_unformatted[1]); + $seeders = $xpath->evaluate(".//td[@class='coll-2 seeds']", $result)[0]->textContent; + $leechers = $xpath->evaluate(".//td[@class='coll-3 leeches']", $result)[0]->textContent; + + array_push($results, + array ( + "name" => htmlspecialchars($name), + "seeders" => (int) $seeders, + "leechers" => (int) $leechers, + "magnet" => htmlspecialchars($magnet), + "size" => htmlspecialchars($size), + "source" => "1337x.to" + ) + ); + } + + return $results; + } } ?> diff --git a/engines/bittorrent/merge.php b/engines/bittorrent/merge.php index be4727f..ec99127 100644 --- a/engines/bittorrent/merge.php +++ b/engines/bittorrent/merge.php @@ -4,68 +4,32 @@ $this->query = $query; $this->page = $page; - // TODO make these engine requests require "engines/bittorrent/thepiratebay.php"; require "engines/bittorrent/rutor.php"; - require "engines/bittorrent/nyaa.php"; require "engines/bittorrent/yts.php"; require "engines/bittorrent/torrentgalaxy.php"; require "engines/bittorrent/1337x.php"; require "engines/bittorrent/sukebei.php"; - $query = urlencode($query); - - $torrent_urls = array( - $thepiratebay_url, - $rutor_url, - $nyaa_url, - $yts_url, - $torrentgalaxy_url, - $_1337x_url, - $sukebei_url + $this->requests = array( + new PirateBayRequest($query, $page, $mh, $config), + new _1337xRequest($query, $page, $mh, $config), + new NyaaRequest($query, $page, $mh, $config), + new RutorRequest($query, $page, $mh, $config), + new SukebeiRequest($query, $page, $mh, $config), + new TorrentGalaxyRequest($query, $page, $mh, $config), + new YTSRequest($query, $page, $mh, $config), ); - - $this->chs = array(); - - foreach ($torrent_urls as $url) - { - $ch = curl_init($url); - curl_setopt_array($ch, $config->curl_settings); - array_push($this->chs, $ch); - curl_multi_add_handle($mh, $ch); - } } public function get_results() { $query = urlencode($this->query); $results = array(); - for ($i=0; count($this->chs)>$i; $i++) - { - $response = curl_multi_getcontent($this->chs[$i]); - - switch ($i) - { - case 0: - $results = array_merge($results, get_thepiratebay_results($response)); - break; - case 1: - $results = array_merge($results, get_rutor_results($response)); - break; - case 2: - $results = array_merge($results, get_nyaa_results($response)); - break; - case 3: - $results = array_merge($results, get_yts_results($response)); - break; - case 4: - //$results = array_merge($results, get_torrentgalaxy_results($response)); - break; - case 5: - //$results = array_merge($results, get_1337x_results($response)); - break; - case 6: - $results = array_merge($results, get_sukebei_results($response)); - break; + foreach ($this->requests as $request) { + error_log($request->get_request_url()); + error_log( curl_getinfo($request->ch)['http_code'] ); + if ($request->successful()) { + $results = array_merge($results, $request->get_results()); } } @@ -78,30 +42,29 @@ public static function print_results($results) { echo "
"; - if (!empty($results)) - { - foreach($results as $result) - { - $source = $result["source"]; - $name = $result["name"]; - $magnet = $result["magnet"]; - $seeders = $result["seeders"]; - $leechers = $result["leechers"]; - $size = $result["size"]; - - echo "
"; - echo ""; - echo "$source"; - echo "

$name

"; - echo ""; - echo "SE: $seeders - "; - echo "LE: $leechers - "; - echo "$size"; - echo "
"; - } - } - else + if (empty($results)) { echo "

There are no results. Please try different keywords!

"; + return; + } + + foreach($results as $result) { + $source = $result["source"]; + $name = $result["name"]; + $magnet = $result["magnet"]; + $seeders = $result["seeders"]; + $leechers = $result["leechers"]; + $size = $result["size"]; + + echo "
"; + echo ""; + echo "$source"; + echo "

$name

"; + echo ""; + echo "SE: $seeders - "; + echo "LE: $leechers - "; + echo "$size"; + echo "
"; + } echo "
"; } diff --git a/engines/bittorrent/nyaa.php b/engines/bittorrent/nyaa.php index e2b846d..9eeb58d 100644 --- a/engines/bittorrent/nyaa.php +++ b/engines/bittorrent/nyaa.php @@ -1,35 +1,50 @@ query("//tbody/tr") as $result) - { - $name = $xpath->evaluate(".//td[@colspan='2']//a[not(contains(@class, 'comments'))]/@title", $result)[0]->textContent; - $centered = $xpath->evaluate(".//td[@class='text-center']", $result); - $magnet = $xpath->evaluate(".//a[2]/@href", $centered[0])[0]->textContent; - $magnet_without_tracker = explode("&tr=", $magnet)[0]; - $magnet = $magnet_without_tracker . $config->bittorent_trackers; - $size = $centered[1]->textContent; - $seeders = $centered[3]->textContent; - $leechers = $centered[4]->textContent; - - array_push($results, - array ( - "name" => htmlspecialchars($name), - "seeders" => (int) $seeders, - "leechers" => (int) $leechers, - "magnet" => htmlspecialchars($magnet), - "size" => htmlspecialchars($size), - "source" => "nyaa.si" - ) - ); + public function get_request_url() { + return "https://$this->SOURCE/?q=" . urlencode($this->query); } - return $results; + public function get_results() { + $response = curl_multi_getcontent($this->ch); + $xpath = get_xpath($response); + $results = array(); + + foreach($xpath->query("//tbody/tr") as $result) + { + $name_node = $xpath->evaluate(".//td[@colspan='2']//a[not(contains(@class, 'comments'))]/@title", $result); + if ($name_node->length > 0) { + $name = $name_node[0]->textContent; + } else { + $name = ""; + } + $centered = $xpath->evaluate(".//td[@class='text-center']", $result); + $magnet_node = $xpath->evaluate(".//a[2]/@href", $centered[0]); + if ($magnet_node->length > 0) { + $magnet = $magnet_node[0]->textContent; + $magnet_without_tracker = explode("&tr=", $magnet)[0]; + $magnet = $magnet_without_tracker . $this->config->bittorent_trackers; + } else { + $magnet = ""; + } + $size = $centered[1]->textContent; + $seeders = $centered[3]->textContent; + $leechers = $centered[4]->textContent; + + array_push($results, + array ( + "name" => htmlspecialchars($name), + "seeders" => (int) $seeders, + "leechers" => (int) $leechers, + "magnet" => htmlspecialchars($magnet), + "size" => htmlspecialchars($size), + "source" => $this->SOURCE + ) + ); + } + + return $results; + } } ?> diff --git a/engines/bittorrent/rutor.php b/engines/bittorrent/rutor.php index 8369f3e..3bda93f 100644 --- a/engines/bittorrent/rutor.php +++ b/engines/bittorrent/rutor.php @@ -1,36 +1,39 @@ query("//table/tr[@class='gai' or @class='tum']") as $result) - { - $name = $xpath->evaluate(".//td/a", $result)[2]->textContent; - $magnet = $xpath->evaluate(".//td/a/@href", $result)[1]->textContent; - $magnet_without_tracker = explode("&tr=", $magnet)[0]; - $magnet = $magnet_without_tracker . $config->bittorent_trackers; - $td = $xpath->evaluate(".//td", $result); - $size = $td[count($td) == 5 ? 3 : 2]->textContent; - $seeders = $xpath->evaluate(".//span", $result)[0]->textContent; - $leechers = $xpath->evaluate(".//span", $result)[1]->textContent; - - array_push($results, - array ( - "name" => htmlspecialchars($name), - "seeders" => (int) filter_var($seeders, FILTER_SANITIZE_NUMBER_INT), - "leechers" => (int) filter_var($leechers, FILTER_SANITIZE_NUMBER_INT), - "magnet" => htmlspecialchars($magnet), - "size" => htmlspecialchars($size), - "source" => "rutor.info" - ) - ); + class RutorRequest extends EngineRequest { + public function get_request_url() { + return "http://rutor.info/search/" . urlencode($this->query); } - return $results; + public function get_results() { + $response = curl_multi_getcontent($this->ch); + $xpath = get_xpath($response); + $results = array(); + + + foreach($xpath->query("//table/tr[@class='gai' or @class='tum']") as $result) + { + $name = $xpath->evaluate(".//td/a", $result)[2]->textContent; + $magnet = $xpath->evaluate(".//td/a/@href", $result)[1]->textContent; + $magnet_without_tracker = explode("&tr=", $magnet)[0]; + $magnet = $magnet_without_tracker . $this->config->bittorent_trackers; + $td = $xpath->evaluate(".//td", $result); + $size = $td[count($td) == 5 ? 3 : 2]->textContent; + $seeders = $xpath->evaluate(".//span", $result)[0]->textContent; + $leechers = $xpath->evaluate(".//span", $result)[1]->textContent; + + array_push($results, + array ( + "name" => htmlspecialchars($name), + "seeders" => (int) filter_var($seeders, FILTER_SANITIZE_NUMBER_INT), + "leechers" => (int) filter_var($leechers, FILTER_SANITIZE_NUMBER_INT), + "magnet" => htmlspecialchars($magnet), + "size" => htmlspecialchars($size), + "source" => "rutor.info" + ) + ); + } + + return $results; + } } ?> diff --git a/engines/bittorrent/sukebei.php b/engines/bittorrent/sukebei.php index 61485cc..106ff1a 100644 --- a/engines/bittorrent/sukebei.php +++ b/engines/bittorrent/sukebei.php @@ -1,44 +1,6 @@ query("//tbody/tr") as $result) - { - $name_node = $xpath->evaluate(".//td[@colspan='2']//a[not(contains(@class, 'comments'))]/@title", $result); - if ($name_node->length > 0) { - $name = $name_node[0]->textContent; - } else { - $name = ""; - } - $centered = $xpath->evaluate(".//td[@class='text-center']", $result); - $magnet_node = $xpath->evaluate(".//a[2]/@href", $centered[0]); - if ($magnet_node->length > 0) { - $magnet = $magnet_node[0]->textContent; - $magnet_without_tracker = explode("&tr=", $magnet)[0]; - $magnet = $magnet_without_tracker . $config->bittorent_trackers; - } else { - $magnet = ""; - } - $size = $centered[1]->textContent; - $seeders = $centered[3]->textContent; - $leechers = $centered[4]->textContent; - - array_push($results, - array ( - "name" => htmlspecialchars($name), - "seeders" => (int) $seeders, - "leechers" => (int) $leechers, - "magnet" => htmlspecialchars($magnet), - "size" => htmlspecialchars($size), - "source" => "sukebei.nyaa.si" - ) - ); - } - return $results; + include "engines/bittorrent/nyaa.php"; + class SukebeiRequest extends NyaaRequest { + public $SOURCE = "sukebei.nyaa.si"; } ?> diff --git a/engines/bittorrent/thepiratebay.php b/engines/bittorrent/thepiratebay.php index 49b2476..02375ec 100644 --- a/engines/bittorrent/thepiratebay.php +++ b/engines/bittorrent/thepiratebay.php @@ -1,44 +1,46 @@ query); + } - $thepiratebay_url = "https://apibay.org/q.php?q=$query"; + public function get_results() { + $response = curl_multi_getcontent($this->ch); + $results = array(); + $json_response = json_decode($response, true); - function get_thepiratebay_results($response) - { - global $config; - $results = array(); - $json_response = json_decode($response, true); + if (empty($json_response)) + { + return $results; + } + + foreach ($json_response as $response) + { + $size = human_filesize($response["size"]); + $hash = $response["info_hash"]; + $name = $response["name"]; + $seeders = (int) $response["seeders"]; + $leechers = (int) $response["leechers"]; + + $magnet = "magnet:?xt=urn:btih:$hash&dn=$name" . $config->bittorent_trackers; + + if ($name == "No results returned") + break; + + array_push($results, + array ( + "size" => htmlspecialchars($size), + "name" => htmlspecialchars($name), + "seeders" => (int) htmlspecialchars($seeders), + "leechers" => (int) htmlspecialchars($leechers), + "magnet" => htmlspecialchars($magnet), + "source" => "thepiratebay.org" + ) + ); + } - if (empty($json_response)) - { return $results; + } - - foreach ($json_response as $response) - { - $size = human_filesize($response["size"]); - $hash = $response["info_hash"]; - $name = $response["name"]; - $seeders = (int) $response["seeders"]; - $leechers = (int) $response["leechers"]; - - $magnet = "magnet:?xt=urn:btih:$hash&dn=$name" . $config->bittorent_trackers; - - if ($name == "No results returned") - break; - - array_push($results, - array ( - "size" => htmlspecialchars($size), - "name" => htmlspecialchars($name), - "seeders" => (int) htmlspecialchars($seeders), - "leechers" => (int) htmlspecialchars($leechers), - "magnet" => htmlspecialchars($magnet), - "source" => "thepiratebay.org" - ) - ); - } - - return $results; - } ?> diff --git a/engines/bittorrent/torrentgalaxy.php b/engines/bittorrent/torrentgalaxy.php index 5d31e82..75c209b 100644 --- a/engines/bittorrent/torrentgalaxy.php +++ b/engines/bittorrent/torrentgalaxy.php @@ -1,34 +1,38 @@ query("//div[@class='tgxtablerow txlight']") as $result) - { - $name = $xpath->evaluate(".//div[contains(@class, 'clickable-row')]", $result)[0]->textContent; - $magnet = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/a/@href", $result)[1]->textContent; - $magnet_without_tracker = explode("&tr=", $magnet)[0]; - $magnet = $magnet_without_tracker . $config->bittorent_trackers; - $size = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span", $result)[0]->textContent; - $seeders = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[1]->textContent; - $leechers = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[2]->textContent; - - array_push($results, - array ( - "name" => htmlspecialchars($name), - "seeders" => (int) $seeders, - "leechers" => (int) $leechers, - "magnet" => htmlspecialchars($magnet), - "size" => htmlspecialchars($size), - "source" => "torrentgalaxy.to" - ) - ); + class TorrentGalaxyRequest extends EngineRequest { + public function get_request_url() { + $query = urlencode($this->query); + return "https://torrentgalaxy.to/torrents.php?search=$query#results"; } - return $results; - } + public function get_results() { + $response = curl_multi_getcontent($this->ch); + $xpath = get_xpath($response); + $results = array(); + + foreach($xpath->query("//div[@class='tgxtablerow txlight']") as $result) + { + $name = $xpath->evaluate(".//div[contains(@class, 'clickable-row')]", $result)[0]->textContent; + $magnet = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/a/@href", $result)[1]->textContent; + $magnet_without_tracker = explode("&tr=", $magnet)[0]; + $magnet = $magnet_without_tracker . $this->config->bittorent_trackers; + $size = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span", $result)[0]->textContent; + $seeders = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[1]->textContent; + $leechers = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[2]->textContent; + + array_push($results, + array ( + "name" => htmlspecialchars($name), + "seeders" => (int) $seeders, + "leechers" => (int) $leechers, + "magnet" => htmlspecialchars($magnet), + "size" => htmlspecialchars($size), + "source" => "torrentgalaxy.to" + ) + ); + } + + return $results; + } + } ?> diff --git a/engines/bittorrent/yts.php b/engines/bittorrent/yts.php index f3a588d..ea2fd26 100644 --- a/engines/bittorrent/yts.php +++ b/engines/bittorrent/yts.php @@ -1,45 +1,47 @@ bittorent_trackers"; - - array_push($results, - array ( - "size" => htmlspecialchars($size), - "name" => htmlspecialchars($name), - "seeders" => htmlspecialchars($seeders), - "leechers" => htmlspecialchars($leechers), - "magnet" => htmlspecialchars($magnet), - "source" => "yts.mx" - ) - ); - - } - } + class YTSRequest extends EngineRequest { + public function get_request_url() { + return "https://yts.mx/api/v2/list_movies.json?query_term=" . urlencode($this->query); } - return $results; - + public function get_results() { + $response = curl_multi_getcontent($this->ch); + global $config; + $results = array(); + $json_response = json_decode($response, true); + + if ($json_response["status"] == "ok" && $json_response["data"]["movie_count"] != 0) + { + foreach ($json_response["data"]["movies"] as $movie) + { + $name = $movie["title"]; + $name_encoded = urlencode($name); + + foreach ($movie["torrents"] as $torrent) + { + + $hash = $torrent["hash"]; + $seeders = $torrent["seeds"]; + $leechers = $torrent["peers"]; + $size = $torrent["size"]; + + $magnet = "magnet:?xt=urn:btih:$hash&dn=$name_encoded$config->bittorent_trackers"; + + array_push($results, + array ( + "size" => htmlspecialchars($size), + "name" => htmlspecialchars($name), + "seeders" => htmlspecialchars($seeders), + "leechers" => htmlspecialchars($leechers), + "magnet" => htmlspecialchars($magnet), + "source" => "yts.mx" + ) + ); + } + } + } + + return $results; + } } ?> diff --git a/misc/search_engine.php b/misc/search_engine.php index 28a8b26..c028913 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -17,6 +17,10 @@ return ""; } + public function successful() { + return curl_getinfo($this->ch)['http_code'] == '200'; + } + abstract function get_results(); static public function print_results($results){} } From 6c253bcb478a1625d350033534ae49b0c40a38dc Mon Sep 17 00:00:00 2001 From: davidovski Date: Tue, 22 Aug 2023 18:41:02 +0100 Subject: [PATCH 09/40] put all settings in single "opts" array --- config.php.example | 10 ++--- engines/ahmia/hidden_service.php | 5 +-- engines/bittorrent/1337x.php | 4 +- engines/bittorrent/merge.php | 25 +++++-------- engines/bittorrent/nyaa.php | 2 +- engines/bittorrent/rutor.php | 2 +- engines/bittorrent/thepiratebay.php | 2 +- engines/bittorrent/torrentgalaxy.php | 2 +- engines/bittorrent/yts.php | 30 +++++++-------- engines/special/currency.php | 39 ++++++++++--------- engines/special/definition.php | 1 + engines/special/ip.php | 19 ++++------ engines/special/special.php | 21 ++++++----- engines/special/user_agent.php | 8 +--- engines/special/wikipedia.php | 12 +++--- engines/text/duckduckgo.php | 8 +--- engines/text/google.php | 11 ++---- engines/text/text.php | 24 ++++++------ misc/header.php | 6 +-- misc/search_engine.php | 56 +++++++++++++++++++--------- search.php | 27 ++++++-------- settings.php | 1 + 22 files changed, 151 insertions(+), 164 deletions(-) diff --git a/config.php.example b/config.php.example index 99ad26d..9d28718 100644 --- a/config.php.example +++ b/config.php.example @@ -4,13 +4,9 @@ // e.g.: fr -> https://google.fr/ "google_domain" => "com", - // Google results will be in this language - "google_language_site" => "", - "google_language_results" => "", - "google_number_of_results" => 10, - - // You can set a language for results in wikipedia - "wikipedia_language" => "en", + // Results will be in this language + "language" => "", + "number_of_results" => 10, // You can use any Invidious instance here "invidious_instance_for_video_results" => "https://invidious.snopyta.org", diff --git a/engines/ahmia/hidden_service.php b/engines/ahmia/hidden_service.php index e88da58..9ddd58b 100644 --- a/engines/ahmia/hidden_service.php +++ b/engines/ahmia/hidden_service.php @@ -1,10 +1,9 @@ query); - return "https://ahmia.fi/search/?q=$query"; + return "https://ahmia.fi/search/?q=" . urlencode($this->query); } public function get_results() { diff --git a/engines/bittorrent/1337x.php b/engines/bittorrent/1337x.php index e7ae3e5..c70468f 100644 --- a/engines/bittorrent/1337x.php +++ b/engines/bittorrent/1337x.php @@ -11,9 +11,7 @@ $xpath = get_xpath($response); $results = array(); - foreach($xpath->query("//table/tbody/tr") as $result) - { - + foreach($xpath->query("//table/tbody/tr") as $result) { $name = $xpath->evaluate(".//td[@class='coll-1 name']/a", $result)[1]->textContent; $magnet = "./engines/bittorrent/get_magnet_1337x.php?url=https://1337x.to" . $xpath->evaluate(".//td[@class='coll-1 name']/a/@href", $result)[1]->textContent; $size_unformatted = explode(" ", $xpath->evaluate(".//td[contains(@class, 'coll-4 size')]", $result)[0]->textContent); diff --git a/engines/bittorrent/merge.php b/engines/bittorrent/merge.php index ec99127..31dcc24 100644 --- a/engines/bittorrent/merge.php +++ b/engines/bittorrent/merge.php @@ -1,8 +1,7 @@ query = $query; - $this->page = $page; + public function __construct($opts, $mh) { + parent::__construct($opts, $mh); require "engines/bittorrent/thepiratebay.php"; require "engines/bittorrent/rutor.php"; @@ -12,25 +11,21 @@ require "engines/bittorrent/sukebei.php"; $this->requests = array( - new PirateBayRequest($query, $page, $mh, $config), - new _1337xRequest($query, $page, $mh, $config), - new NyaaRequest($query, $page, $mh, $config), - new RutorRequest($query, $page, $mh, $config), - new SukebeiRequest($query, $page, $mh, $config), - new TorrentGalaxyRequest($query, $page, $mh, $config), - new YTSRequest($query, $page, $mh, $config), + new PirateBayRequest($opts, $mh), + new _1337xRequest($opts, $mh), + new NyaaRequest($opts, $mh), + new RutorRequest($opts, $mh), + new SukebeiRequest($opts, $mh), + new TorrentGalaxyRequest($opts, $mh), + new YTSRequest($opts, $mh), ); } public function get_results() { - $query = urlencode($this->query); $results = array(); foreach ($this->requests as $request) { - error_log($request->get_request_url()); - error_log( curl_getinfo($request->ch)['http_code'] ); - if ($request->successful()) { + if ($request->successful()) $results = array_merge($results, $request->get_results()); - } } $seeders = array_column($results, "seeders"); diff --git a/engines/bittorrent/nyaa.php b/engines/bittorrent/nyaa.php index 9eeb58d..c250f1a 100644 --- a/engines/bittorrent/nyaa.php +++ b/engines/bittorrent/nyaa.php @@ -24,7 +24,7 @@ if ($magnet_node->length > 0) { $magnet = $magnet_node[0]->textContent; $magnet_without_tracker = explode("&tr=", $magnet)[0]; - $magnet = $magnet_without_tracker . $this->config->bittorent_trackers; + $magnet = $magnet_without_tracker . $this->opts->bittorent_trackers; } else { $magnet = ""; } diff --git a/engines/bittorrent/rutor.php b/engines/bittorrent/rutor.php index 3bda93f..f6790a7 100644 --- a/engines/bittorrent/rutor.php +++ b/engines/bittorrent/rutor.php @@ -15,7 +15,7 @@ $name = $xpath->evaluate(".//td/a", $result)[2]->textContent; $magnet = $xpath->evaluate(".//td/a/@href", $result)[1]->textContent; $magnet_without_tracker = explode("&tr=", $magnet)[0]; - $magnet = $magnet_without_tracker . $this->config->bittorent_trackers; + $magnet = $magnet_without_tracker . $this->opts->bittorrent_trackers; $td = $xpath->evaluate(".//td", $result); $size = $td[count($td) == 5 ? 3 : 2]->textContent; $seeders = $xpath->evaluate(".//span", $result)[0]->textContent; diff --git a/engines/bittorrent/thepiratebay.php b/engines/bittorrent/thepiratebay.php index 02375ec..49f3f7a 100644 --- a/engines/bittorrent/thepiratebay.php +++ b/engines/bittorrent/thepiratebay.php @@ -22,7 +22,7 @@ $seeders = (int) $response["seeders"]; $leechers = (int) $response["leechers"]; - $magnet = "magnet:?xt=urn:btih:$hash&dn=$name" . $config->bittorent_trackers; + $magnet = "magnet:?xt=urn:btih:$hash&dn=$name" . $this->opts->bittorrent_trackers; if ($name == "No results returned") break; diff --git a/engines/bittorrent/torrentgalaxy.php b/engines/bittorrent/torrentgalaxy.php index 75c209b..d893db3 100644 --- a/engines/bittorrent/torrentgalaxy.php +++ b/engines/bittorrent/torrentgalaxy.php @@ -15,7 +15,7 @@ $name = $xpath->evaluate(".//div[contains(@class, 'clickable-row')]", $result)[0]->textContent; $magnet = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/a/@href", $result)[1]->textContent; $magnet_without_tracker = explode("&tr=", $magnet)[0]; - $magnet = $magnet_without_tracker . $this->config->bittorent_trackers; + $magnet = $magnet_without_tracker . $this->opts->bittorrent_trackers; $size = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span", $result)[0]->textContent; $seeders = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[1]->textContent; $leechers = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[2]->textContent; diff --git a/engines/bittorrent/yts.php b/engines/bittorrent/yts.php index ea2fd26..d146ba7 100644 --- a/engines/bittorrent/yts.php +++ b/engines/bittorrent/yts.php @@ -10,24 +10,25 @@ $results = array(); $json_response = json_decode($response, true); - if ($json_response["status"] == "ok" && $json_response["data"]["movie_count"] != 0) + if ($json_response["status"] != "ok" || $json_response["data"]["movie_count"] == 0) + return $results; + + foreach ($json_response["data"]["movies"] as $movie) { - foreach ($json_response["data"]["movies"] as $movie) - { - $name = $movie["title"]; - $name_encoded = urlencode($name); + $name = $movie["title"]; + $name_encoded = urlencode($name); - foreach ($movie["torrents"] as $torrent) - { + foreach ($movie["torrents"] as $torrent) + { - $hash = $torrent["hash"]; - $seeders = $torrent["seeds"]; - $leechers = $torrent["peers"]; - $size = $torrent["size"]; + $hash = $torrent["hash"]; + $seeders = $torrent["seeds"]; + $leechers = $torrent["peers"]; + $size = $torrent["size"]; - $magnet = "magnet:?xt=urn:btih:$hash&dn=$name_encoded$config->bittorent_trackers"; + $magnet = "magnet:?xt=urn:btih:$hash&dn=$name_encoded$this->opts->bittorrent_trackers"; - array_push($results, + array_push($results, array ( "size" => htmlspecialchars($size), "name" => htmlspecialchars($name), @@ -37,8 +38,7 @@ "source" => "yts.mx" ) ); - } - } + } } return $results; diff --git a/engines/special/currency.php b/engines/special/currency.php index fd11b79..ba53598 100644 --- a/engines/special/currency.php +++ b/engines/special/currency.php @@ -1,25 +1,24 @@ ch); - - $split_query = explode(" ", $this->query); - - $base_currency = strtoupper($split_query[1]); - $currency_to_convert = strtoupper($split_query[3]); - $amount_to_convert = floatval($split_query[0]); + class CurrencyRequest extends EngineRequest { + public function get_request_url() { + return "https://cdn.moneyconvert.net/api/latest.json"; + } - $json_response = json_decode($response, true); - - $rates = $json_response["rates"]; + public function get_results() { + $response = curl_multi_getcontent($this->ch); - if (array_key_exists($base_currency, $rates) && array_key_exists($currency_to_convert, $rates)) - { + $split_query = explode(" ", $this->query); + + $base_currency = strtoupper($split_query[1]); + $currency_to_convert = strtoupper($split_query[3]); + $amount_to_convert = floatval($split_query[0]); + + $json_response = json_decode($response, true); + + $rates = $json_response["rates"]; + + if (!array_key_exists($base_currency, $rates) || !array_key_exists($currency_to_convert, $rates)) + return array(); $base_currency_response = $rates[$base_currency]; $currency_to_convert_response = $rates[$currency_to_convert]; @@ -33,6 +32,6 @@ class CurrencyRequest extends EngineRequest { "source" => $source ) ); - } + } } ?> diff --git a/engines/special/definition.php b/engines/special/definition.php index 819eeb4..b862b32 100644 --- a/engines/special/definition.php +++ b/engines/special/definition.php @@ -1,5 +1,6 @@ query); $reversed_split_q = array_reverse($split_query); diff --git a/engines/special/ip.php b/engines/special/ip.php index 1e7cab9..6fbb684 100644 --- a/engines/special/ip.php +++ b/engines/special/ip.php @@ -1,17 +1,12 @@ query = $query; - } - - function get_results() - { - return array( - "special_response" => array( - "response" => $_SERVER["REMOTE_ADDR"], - "source" => null - ) - ); + function get_results() { + return array( + "special_response" => array( + "response" => $_SERVER["REMOTE_ADDR"], + "source" => null + ) + ); } } ?> diff --git a/engines/special/special.php b/engines/special/special.php index d149ee2..41f706f 100644 --- a/engines/special/special.php +++ b/engines/special/special.php @@ -44,8 +44,11 @@ return 0; } - function get_special_search_request ($query, $page, $mh, $config) { - $special_search = $page ? 0 : check_for_special_search($query); + function get_special_search_request($opts, $mh) { + if ($opts->page != 0) + return null; + + $special_search = check_for_special_search($opts->query); $special_request = null; $url = null; @@ -55,31 +58,31 @@ switch ($special_search) { case 1: require "engines/special/currency.php"; - $special_request = new CurrencyRequest($query, $page, $mh, $config); + $special_request = new CurrencyRequest($opts, $mh); break; case 2: require "engines/special/definition.php"; - $special_request = new DefinitionRequest($query, $page, $mh, $config); + $special_request = new DefinitionRequest($opts, $mh); break; case 3: require "engines/special/ip.php"; - $special_request = new IPRequest($query, $page, $mh, $config); + $special_request = new IPRequest($opts, $mh); break; case 4: require "engines/special/user_agent.php"; - $special_request = new UserAgentRequest($query, $page, $mh, $config); + $special_request = new UserAgentRequest($opts, $mh); break; case 5: require "engines/special/weather.php"; - $special_request = new WeatherRequest($query, $page, $mh, $config); + $special_request = new WeatherRequest($opts, $mh); break; case 6: require "engines/special/tor.php"; - $special_request = new TorRequest($query, $page, $mh, $config); + $special_request = new TorRequest($opts, $mh); break; case 7: require "engines/special/wikipedia.php"; - $special_request = new WikipediaRequest($query, $page, $mh, $config); + $special_request = new WikipediaRequest($opts, $mh); break; } diff --git a/engines/special/user_agent.php b/engines/special/user_agent.php index aab3d59..64f5e8a 100644 --- a/engines/special/user_agent.php +++ b/engines/special/user_agent.php @@ -1,12 +1,6 @@ query = $query; - $this->page = $page; - } - - function get_results() - { + function get_results() { return array( "special_response" => array( "response" => $_SERVER["HTTP_USER_AGENT"], diff --git a/engines/special/wikipedia.php b/engines/special/wikipedia.php index 8f5189e..786a357 100644 --- a/engines/special/wikipedia.php +++ b/engines/special/wikipedia.php @@ -1,13 +1,13 @@ wikipedia_language = isset($_COOKIE["wikipedia_language"]) ? trim(htmlspecialchars($_COOKIE["wikipedia_language"])) : $this->config->wikipedia_language; + $this->wikipedia_language = $this->opts->language; $query_encoded = urlencode($this->query); - if (in_array($this->wikipedia_language, json_decode(file_get_contents("static/misc/wikipedia_langs.json"), true))) - return "https://$this->wikipedia_language.wikipedia.org/w/api.php?format=json&action=query&prop=extracts%7Cpageimages&exintro&explaintext&redirects=1&pithumbsize=500&titles=$query_encoded"; + if (!in_array($this->wikipedia_language, json_decode(file_get_contents("static/misc/wikipedia_langs.json"), true))) + $this->wikipedia_language = "en"; - return ""; + return "https://$this->wikipedia_language.wikipedia.org/w/api.php?format=json&action=query&prop=extracts%7Cpageimages&exintro&explaintext&redirects=1&pithumbsize=500&titles=$query_encoded"; } public function get_results() { @@ -19,6 +19,7 @@ if (array_key_exists("missing", $first_page)) return array(); + $description = substr($first_page["extract"], 0, 250) . "..."; $source = check_for_privacy_frontend("https://$this->wikipedia_language.wikipedia.org/wiki/$this->query"); @@ -29,8 +30,7 @@ ) ); - if (array_key_exists("thumbnail", $first_page)) - { + if (array_key_exists("thumbnail", $first_page)) { $image_url = $first_page["thumbnail"]["source"]; $response["special_response"]["image"] = $image_url; } diff --git a/engines/text/duckduckgo.php b/engines/text/duckduckgo.php index a781710..6833f4a 100644 --- a/engines/text/duckduckgo.php +++ b/engines/text/duckduckgo.php @@ -5,15 +5,11 @@ $query_encoded = str_replace("%22", "\"", urlencode($this->query)); $results = array(); - // $domain = $this->config->google_domain; $domain = 'com'; - $site_language = isset($_COOKIE["google_language_site"]) ? trim(htmlspecialchars($_COOKIE["google_language_site"])) : $this->config->google_language_site; - $results_language = isset($_COOKIE["google_language_results"]) ? trim(htmlspecialchars($_COOKIE["google_language_results"])) : $this->config->google_language_results; - $number_of_results = isset($_COOKIE["google_number_of_results"]) ? trim(htmlspecialchars($_COOKIE["google_number_of_results"])) : $this->config->google_number_of_results; + $results_language = $this->opts->language; + $number_of_results = $this->opts->number_of_results; $url = "https://html.duckduckgo.$domain/html/?q=$query_encoded&kd=-1&s=" . 3 * $this->page; - if (3 > strlen($site_language) && 0 < strlen($site_language)) - $url .= "&hl=$site_language"; if (3 > strlen($results_language) && 0 < strlen($results_language)) $url .= "&lr=lang_$results_language"; diff --git a/engines/text/google.php b/engines/text/google.php index eb2ac26..83c731f 100644 --- a/engines/text/google.php +++ b/engines/text/google.php @@ -6,16 +6,11 @@ $query_encoded = str_replace("%22", "\"", urlencode($this->query)); $results = array(); - $domain = $this->config->google_domain; - $site_language = isset($_COOKIE["google_language_site"]) ? trim(htmlspecialchars($_COOKIE["google_language_site"])) : $this->config->google_language_site; - $results_language = isset($_COOKIE["google_language_results"]) ? trim(htmlspecialchars($_COOKIE["google_language_results"])) : $this->config->google_language_results; - $number_of_results = isset($_COOKIE["google_number_of_results"]) ? trim(htmlspecialchars($_COOKIE["google_number_of_results"])) : $this->config->google_number_of_results; + $domain = $this->opts->google_domain; + $results_language = $this->opts->language; + $number_of_results = $this->opts->number_of_results; $url = "https://www.google.$domain/search?q=$query_encoded&nfpr=1&start=$this->page"; - error_log($url); - - if (3 > strlen($site_language) && 0 < strlen($site_language)) - $url .= "&hl=$site_language"; if (3 > strlen($results_language) && 0 < strlen($results_language)) $url .= "&lr=lang_$results_language"; diff --git a/engines/text/text.php b/engines/text/text.php index 4a4e0f8..df29438 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -1,29 +1,29 @@ query = $query; - $this->page = $page; + public function __construct($opts, $mh) { + $this->query = $opts->query; + $this->page = $opts->page; + $this->opts = $opts; - $engine=$config->preferred_engines['text']; - if (is_null($engine)) - $engine = "google"; - $query_parts = explode(" ", $query); + $engine = $opts->preferred_engines->text ?? "google"; + + $query_parts = explode(" ", $this->query); $last_word_query = end($query_parts); - if (substr($query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!") - check_ddg_bang($query); + if (substr($this->query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!") + check_ddg_bang($this->query); if ($engine == "google") { require "engines/text/google.php"; - $this->engine_request = new GoogleRequest($query, $page, $mh, $config); + $this->engine_request = new GoogleRequest($opts, $mh); } if ($engine == "duckduckgo") { require "engines/text/duckduckgo.php"; - $this->engine_request = new DuckDuckGoRequest($query, $page, $mh, $config); + $this->engine_request = new DuckDuckGoRequest($opts, $mh); } require "engines/special/special.php"; - $this->special_request = get_special_search_request($query, $page, $mh, $config); + $this->special_request = get_special_search_request($opts, $mh); } public function get_results() { diff --git a/misc/header.php b/misc/header.php index 657fa67..2d6db46 100644 --- a/misc/header.php +++ b/misc/header.php @@ -8,9 +8,5 @@ "/> diff --git a/misc/search_engine.php b/misc/search_engine.php index c028913..dd877de 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -1,14 +1,14 @@ query = $query; - $this->page = $page; - $this->config = $config; + function __construct($opts, $mh) { + $this->query = $opts->query; + $this->page = $opts->page; + $this->opts = $opts; $url = $this->get_request_url(); if ($url) { $this->ch = curl_init($url); - curl_setopt_array($this->ch, $config->curl_settings); + curl_setopt_array($this->ch, $opts->curl_settings); curl_multi_add_handle($mh, $this->ch); } } @@ -25,44 +25,64 @@ static public function print_results($results){} } - function init_search($type, $query, $page, $mh, $config) { - switch ($type) + function load_opts() { + $opts = require "config.php"; + + $opts->query = trim($_REQUEST["q"]); + $opts->type = (int) $_REQUEST["t"] ?? 0; + $opts->page = (int) $_REQUEST["p"] ?? 0; + + $opts->theme = trim(htmlspecialchars($_COOKIE["theme"] ?? "dark")); + $opts->safe_search = isset($_COOKIE["safe_search"]); + $opts->disable_special = isset($_COOKIE["disable_special"]); + $opts->disable_frontends = isset($_COOKIE["disable_frontends"]); + + $opts->language ??= trim(htmlspecialchars($_COOKIE["language"])); + $opts->number_of_results ??= trim(htmlspecialchars($_COOKIE["number_of_results"])); + + // TODO frontends + + return $opts; + } + + function init_search($opts, $mh) { + switch ($opts->type) { case 1: require "engines/qwant/image.php"; - return new QwantImageSearch($query, $page, $mh, $config); + return new QwantImageSearch($opts, $mh); case 2: require "engines/invidious/video.php"; - return new VideoSearch($query, $page, $mh, $config); + return new VideoSearch($opts, $mh); case 3: - if ($config->disable_bittorent_search) { + if ($opts->disable_bittorent_search) { echo "

The host disabled this feature! :C

"; break; } require "engines/bittorrent/merge.php"; - return new TorrentSearch($query, $page, $mh, $config); + return new TorrentSearch($opts, $mh); case 4: - if ($config->disable_hidden_service_search) { + if ($opts->disable_hidden_service_search) { echo "

The host disabled this feature! :C

"; break; } require "engines/ahmia/hidden_service.php"; - return new TorSearch($query, $page, $mh, $config); + return new TorSearch($opts, $mh); default: require "engines/text/text.php"; - return new TextSearch($query, $page, $mh, $config); + return new TextSearch($opts, $mh); } } - function fetch_search_results($type, $query, $page, $config, $do_print) { + function fetch_search_results($opts, $do_print) { $start_time = microtime(true); $mh = curl_multi_init(); - $search_category = init_search($type, $query, $page, $mh, $config); + $search_category = init_search($opts, $mh); $running = null; @@ -70,7 +90,9 @@ curl_multi_exec($mh, $running); } while ($running); - $results = $search_category->get_results($query, $page); + $results = $search_category->get_results(); + + // TODO test if no results here and fallback if (!$do_print) return $results; diff --git a/search.php b/search.php index 07319d3..25497d6 100644 --- a/search.php +++ b/search.php @@ -1,10 +1,11 @@ - 1) return; @@ -23,15 +24,11 @@ echo ""; } - - $query = trim($_REQUEST["q"]); - $type = isset($_REQUEST["t"]) ? (int) $_REQUEST["t"] : 0; - $page = isset($_REQUEST["p"]) ? (int) $_REQUEST["p"] : 0; ?> <?php - echo $query; + echo $opts->query; ?> - LibreY @@ -39,18 +36,18 @@

LibreY

strlen($query) || strlen($query) > 256) + if (1 > strlen($opts->query) || strlen($opts->query) > 256) { header("Location: ./"); die(); } - echo "value=\"" . htmlspecialchars($query) . "\""; + echo "value=\"" . htmlspecialchars($opts->query) . "\""; ?> >
"; + echo ""; ?> @@ -62,13 +59,13 @@ { $category_index = array_search($category, $categories); - if (($config->disable_bittorent_search && $category_index == 3) || - ($config->disable_hidden_service_search && $category_index ==4)) + if (($opts->disable_bittorent_search && $category_index == 3) || + ($opts->disable_hidden_service_search && $category_index ==4)) { continue; } - echo "\""" . ucfirst($category) . ""; + echo "type) ? "class=\"active\" " : "") . "href=\"./search.php?q=" . urlencode($opts->query) . "&p=0&t=" . $category_index . "\">\""" . ucfirst($category) . ""; } ?> @@ -76,8 +73,8 @@ type, $opts->query, $opts->page); ?> diff --git a/settings.php b/settings.php index 5c943f4..e67baa6 100644 --- a/settings.php +++ b/settings.php @@ -1,4 +1,5 @@ Date: Tue, 22 Aug 2023 22:48:37 +0100 Subject: [PATCH 10/40] Make settings.php use opts --- misc/search_engine.php | 13 ++++-- settings.php | 104 +++++++++++++++-------------------------- 2 files changed, 47 insertions(+), 70 deletions(-) diff --git a/misc/search_engine.php b/misc/search_engine.php index dd877de..b6dbb53 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -28,9 +28,9 @@ function load_opts() { $opts = require "config.php"; - $opts->query = trim($_REQUEST["q"]); - $opts->type = (int) $_REQUEST["t"] ?? 0; - $opts->page = (int) $_REQUEST["p"] ?? 0; + $opts->query = trim($_REQUEST->q ?? 0); + $opts->type = (int) ($_REQUEST["t"] ?? 0); + $opts->page = (int) ($_REQUEST["p"] ?? 0); $opts->theme = trim(htmlspecialchars($_COOKIE["theme"] ?? "dark")); $opts->safe_search = isset($_COOKIE["safe_search"]); @@ -40,7 +40,12 @@ $opts->language ??= trim(htmlspecialchars($_COOKIE["language"])); $opts->number_of_results ??= trim(htmlspecialchars($_COOKIE["number_of_results"])); - // TODO frontends + foreach (array_keys($opts->frontends ?? array()) as $frontend) { + $opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? ""; + } + + echo "
";
+        echo "
"; return $opts; } diff --git a/settings.php b/settings.php index e67baa6..6a4f849 100644 --- a/settings.php +++ b/settings.php @@ -1,44 +1,35 @@ $value) - { - if (!empty($value)) - { - setcookie($key, $value, time() + (86400 * 90), '/'); - } - else - { - setcookie($key, "", time() - 1000); - } - } + if (isset($_REQUEST["save"])) { + foreach($_POST as $key=>$value) { + if (!empty($value)) { + setcookie($key, $value, time() + (86400 * 90), '/'); + } else { + setcookie($key, "", time() - 1000); } + } + } - if (isset($_REQUEST["save"]) || isset($_REQUEST["reset"])) - { - header("Location: ./"); - die(); - } + if (isset($_REQUEST["save"]) || isset($_REQUEST["reset"])) { + header("Location: ./"); + die(); + } - require "misc/header.php"; + require "misc/header.php"; ?> LibreY - Settings @@ -68,10 +59,9 @@ "; - if (isset($_COOKIE["theme"])) - { - $cookie_theme = $_COOKIE["theme"]; - $themes = str_replace($cookie_theme . "\"", $cookie_theme . "\" selected", $themes); + if (isset($_COOKIE["theme"])) { + $theme = $opts->theme; + $themes = str_replace($theme . "\"", $theme . "\" selected", $themes); } echo $themes; @@ -80,19 +70,19 @@
- > + disable_special ? "checked" : ""; ?> >

Privacy friendly frontends

For an example if you want to view YouTube without getting spied on, click on "Invidious", find the instance that is most suitable for you then paste it in (correct format: https://example.com)

frontends as $frontend => $data) + foreach($opts->frontends as $frontend => $data) { echo "
"; echo "" . ucfirst($frontend) . ""; echo "frontends["$frontend"]["instance_url"] ?? ""); echo ">"; echo "
"; } @@ -100,43 +90,25 @@
- > + disable_frontends ? "checked" : ""; ?> >

Search settings

- Site language + Language google_language_site; - ?>"> -
-
- Results language - google_language_results; - ?>"> + // TODO make this a dropdown + echo "language ?? "") . "\">"; + ?>
- google_number_of_results; ?>" > + " >
- > -
-
- -

Wikipedia settings

-
-
- Results language - wikipedia_language; - ?>"> + safe_search ? "checked" : ""; ?> >
From b3693add934c3cfb1d8f14991f4deb4bb17e9e41 Mon Sep 17 00:00:00 2001 From: davidovski Date: Tue, 22 Aug 2023 22:49:24 +0100 Subject: [PATCH 11/40] use correct parameter for search query --- misc/search_engine.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/search_engine.php b/misc/search_engine.php index b6dbb53..d62db9a 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -28,7 +28,7 @@ function load_opts() { $opts = require "config.php"; - $opts->query = trim($_REQUEST->q ?? 0); + $opts->query = trim($_REQUEST["q"] ?? ""); $opts->type = (int) ($_REQUEST["t"] ?? 0); $opts->page = (int) ($_REQUEST["p"] ?? 0); From 649b0065e3f7b0a8e83196b354dd6f5ef7b445d3 Mon Sep 17 00:00:00 2001 From: davidovski Date: Tue, 22 Aug 2023 23:55:11 +0100 Subject: [PATCH 12/40] make some options editable via url parameters --- engines/librex/{text.php => fallback.php} | 17 ++++- engines/special/wikipedia.php | 2 +- engines/text/duckduckgo.php | 2 +- engines/text/google.php | 3 +- misc/search_engine.php | 42 +++++++++--- misc/tools.php | 79 +++++++---------------- 6 files changed, 75 insertions(+), 70 deletions(-) rename engines/librex/{text.php => fallback.php} (80%) diff --git a/engines/librex/text.php b/engines/librex/fallback.php similarity index 80% rename from engines/librex/text.php rename to engines/librex/fallback.php index ba314e3..8492bc2 100644 --- a/engines/librex/text.php +++ b/engines/librex/fallback.php @@ -1,7 +1,19 @@ instance = $instance; + parent::__construct($opts, $mh); + } + + public function get_request_url() { + return $instance . "api.php?" .opts_to_params($opts); + } + +} + + + function get_librex_results($opts, $mh) { global $config; if (isset($_REQUEST["nfb"]) && $_REQUEST["nfb"] == "1") @@ -50,4 +62,5 @@ return array_values($results); } + ?> diff --git a/engines/special/wikipedia.php b/engines/special/wikipedia.php index 786a357..436deb4 100644 --- a/engines/special/wikipedia.php +++ b/engines/special/wikipedia.php @@ -22,7 +22,7 @@ $description = substr($first_page["extract"], 0, 250) . "..."; - $source = check_for_privacy_frontend("https://$this->wikipedia_language.wikipedia.org/wiki/$this->query"); + $source = check_for_privacy_frontend("https://$this->wikipedia_language.wikipedia.org/wiki/$this->query", $this->opts); $response = array( "special_response" => array( "response" => htmlspecialchars($description), diff --git a/engines/text/duckduckgo.php b/engines/text/duckduckgo.php index 6833f4a..98026c9 100644 --- a/engines/text/duckduckgo.php +++ b/engines/text/duckduckgo.php @@ -41,7 +41,7 @@ $url = $url->textContent; - $url = check_for_privacy_frontend($url); + $url = check_for_privacy_frontend($url, $opts); $title = $xpath->evaluate(".//h2[@class='result__title']", $result)[0]; $description = $xpath->evaluate(".//a[@class='result__snippet']", $result)[0]; diff --git a/engines/text/google.php b/engines/text/google.php index 83c731f..6104d48 100644 --- a/engines/text/google.php +++ b/engines/text/google.php @@ -51,8 +51,7 @@ } $url = $url->textContent; - - $url = check_for_privacy_frontend($url); + $url = check_for_privacy_frontend($url, $opts); $title = $xpath->evaluate(".//h3", $result)[0]; $description = $xpath->evaluate(".//div[contains(@class, 'VwiC3b')]", $result)[0]; diff --git a/misc/search_engine.php b/misc/search_engine.php index d62db9a..3561a0a 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -31,25 +31,46 @@ $opts->query = trim($_REQUEST["q"] ?? ""); $opts->type = (int) ($_REQUEST["t"] ?? 0); $opts->page = (int) ($_REQUEST["p"] ?? 0); + $opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0; $opts->theme = trim(htmlspecialchars($_COOKIE["theme"] ?? "dark")); - $opts->safe_search = isset($_COOKIE["safe_search"]); - $opts->disable_special = isset($_COOKIE["disable_special"]); - $opts->disable_frontends = isset($_COOKIE["disable_frontends"]); - $opts->language ??= trim(htmlspecialchars($_COOKIE["language"])); + $opts->safe_search = (int) ($_REQUEST["safe"] ?? 0) == 1 || isset($_COOKIE["safe_search"]); + + $opts->disable_special = (int) ($_REQUEST["ns"] ?? 0) == 1 || isset($_COOKIE["disable_special"]); + + $opts->disable_frontends = (int) ($_REQUEST["nf"] ?? 0) == 1 || isset($_COOKIE["disable_frontends"]); + + $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"])); + + $opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0; + if (!$opts->instance_fallback) { + $opts->do_fallback = false; + } + $opts->number_of_results ??= trim(htmlspecialchars($_COOKIE["number_of_results"])); foreach (array_keys($opts->frontends ?? array()) as $frontend) { $opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? ""; } - - echo "
";
-        echo "
"; - return $opts; } + function opts_to_params($opts) { + $query = urlencode($opts->query); + + $params = ""; + $params .= "p=$opts->page"; + $params .= "&q=$query"; + $params .= "&t=$opts->type"; + $params .= "&nfb=" . $opts->do_fallback ? 0 : 1; + $params .= "&safe=" . $opts->safe_search ? 1 : 0; + $params .= "&nf=" . $opts->disable_frontends ? 1 : 0; + $params .= "&ns=" . $opts->disable_special ? 1 : 0; + + return $params; + } + function init_search($opts, $mh) { switch ($opts->type) { @@ -97,7 +118,10 @@ $results = $search_category->get_results(); - // TODO test if no results here and fallback + if (empty($results)) { + //if (!$opt->do_fallback) + //return ; + } if (!$do_print) return $results; diff --git a/misc/tools.php b/misc/tools.php index b70ba0e..23d41bf 100644 --- a/misc/tools.php +++ b/misc/tools.php @@ -20,50 +20,35 @@ function try_replace_with_frontend($url, $frontend, $original) { global $config; - $frontends = $config->frontends; + $frontends = $opts->frontends; + $frontend = $opts->$frontend["instance_url"]; - if (isset($_COOKIE[$frontend]) || !empty($frontends[$frontend]["instance_url"])) - { + if ($frontend) { - if (isset($_COOKIE[$frontend])) - $frontend = $_COOKIE[$frontend]; - else if (!empty($frontends[$frontend]["instance_url"])) - $frontend = $frontends[$frontend]["instance_url"]; if (empty(trim($frontend))) return $url; - if (strpos($url, "wikipedia.org") !== false) - { + if (strpos($url, "wikipedia.org") !== false) { $wiki_split = explode(".", $url); - if (count($wiki_split) > 1) - { + if (count($wiki_split) > 1) { $lang = explode("://", $wiki_split[0])[1]; $url = $frontend . explode($original, $url)[1] . (strpos($url, "?") !== false ? "&" : "?") . "lang=" . $lang; } - } - else if (strpos($url, "fandom.com") !== false) - { + } else if (strpos($url, "fandom.com") !== false) { $fandom_split = explode(".", $url); - if (count($fandom_split) > 1) - { + if (count($fandom_split) > 1) { $wiki_name = explode("://", $fandom_split[0])[1]; $url = $frontend . "/" . $wiki_name . explode($original, $url)[1]; } - } - else if (strpos($url, "gist.github.com") !== false) - { + } else if (strpos($url, "gist.github.com") !== false) { $gist_path = explode("gist.github.com", $url)[1]; $url = $frontend . "/gist" . $gist_path; - } - else if (strpos($url, "stackexchange.com") !== false) - { + } else if (strpos($url, "stackexchange.com") !== false) { $se_domain = explode(".", explode("://", $url)[1])[0]; $se_path = explode("stackexchange.com", $url)[1]; $url = $frontend . "/exchange" . "/" . $se_domain . $se_path; - } - else - { + } else { $url = $frontend . explode($original, $url)[1]; } @@ -74,25 +59,17 @@ return $url; } - function check_for_privacy_frontend($url) - { - - global $config; - - if (isset($_COOKIE["disable_frontends"])) + function check_for_privacy_frontend($url, $opts) { + if ($opts->disable_frontends) return $url; - foreach($config->frontends as $frontend => $data) - { + foreach($opts->frontends as $frontend => $data) { $original = $data["original_url"]; - if (strpos($url, $original)) - { + if (strpos($url, $original)) { $url = try_replace_with_frontend($url, $frontend, $original); break; - } - else if (strpos($url, "stackexchange.com")) - { + } else if (strpos($url, "stackexchange.com")) { $url = try_replace_with_frontend($url, "anonymousoverflow", "stackexchange.com"); break; } @@ -101,8 +78,7 @@ return $url; } - function get_xpath($response) - { + function get_xpath($response) { $htmlDom = new DOMDocument; @$htmlDom->loadHTML($response); $xpath = new DOMXPath($htmlDom); @@ -110,9 +86,8 @@ return $xpath; } - function request($url) - { - global $config; + function request($url) { + $config ??= require "config.php"; $ch = curl_init($url); curl_setopt_array($ch, $config->curl_settings); @@ -121,28 +96,24 @@ return $response; } - function human_filesize($bytes, $dec = 2) - { + function human_filesize($bytes, $dec = 2) { $size = array('B', 'kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'); $factor = floor((strlen($bytes) - 1) / 3); return sprintf("%.{$dec}f ", $bytes / pow(1024, $factor)) . @$size[$factor]; } - function remove_special($string) - { + function remove_special($string) { $string = preg_replace("/[\r\n]+/", "\n", $string); return trim(preg_replace("/\s+/", ' ', $string)); } - function print_elapsed_time($start_time) - { + function print_elapsed_time($start_time) { $end_time = number_format(microtime(true) - $start_time, 2, '.', ''); echo "

Fetched the results in $end_time seconds

"; } - function print_next_page_button($text, $page, $query, $type) - { + function print_next_page_button($text, $page, $query, $type) { echo "
"; echo ""; echo ""; @@ -151,15 +122,13 @@ echo "
"; } - function copy_cookies($curl) - { + function copy_cookies($curl) { if (array_key_exists("HTTP_COOKIE", $_SERVER)) curl_setopt( $curl, CURLOPT_COOKIE, $_SERVER['HTTP_COOKIE'] ); } - function get_country_emote($code) - { + function get_country_emote($code) { $emoji = []; foreach(str_split($code) as $c) { if(($o = ord($c)) > 64 && $o % 32 < 27) { From 1b898c6614b3c164c09e3b2ea9cde4429808cf67 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 23 Aug 2023 00:38:59 +0100 Subject: [PATCH 13/40] fallback on all queries --- api.php | 13 ++----- engines/ahmia/hidden_service.php | 4 +- engines/bittorrent/1337x.php | 3 ++ engines/bittorrent/nyaa.php | 3 ++ engines/bittorrent/rutor.php | 2 + engines/bittorrent/torrentgalaxy.php | 53 ++++++++++++++------------- engines/invidious/video.php | 2 +- engines/librex/fallback.php | 55 +++++++++++++--------------- engines/qwant/image.php | 3 ++ engines/text/duckduckgo.php | 2 +- engines/text/google.php | 2 +- engines/text/text.php | 7 +--- misc/search_engine.php | 24 +++++++----- misc/tools.php | 15 ++++---- 14 files changed, 99 insertions(+), 89 deletions(-) diff --git a/api.php b/api.php index a18046e..8084fd9 100644 --- a/api.php +++ b/api.php @@ -1,10 +1,10 @@ query) { echo "

Example API request: ./api.php?q=gentoo&p=2&t=0


\"q\" is the keyword

@@ -17,12 +17,7 @@ die(); } - $query = $_REQUEST["q"]; - $query_encoded = urlencode($query); - $page = isset($_REQUEST["p"]) ? (int) $_REQUEST["p"] : 0; - $type = isset($_REQUEST["t"]) ? (int) $_REQUEST["t"] : 0; - - $results = fetch_search_results($type, $query, $page, $config, false); + $results = fetch_search_results($opts, false); header("Content-Type: application/json"); echo json_encode($results); ?> diff --git a/engines/ahmia/hidden_service.php b/engines/ahmia/hidden_service.php index 9ddd58b..f950ba8 100644 --- a/engines/ahmia/hidden_service.php +++ b/engines/ahmia/hidden_service.php @@ -8,9 +8,11 @@ public function get_results() { $response = curl_multi_getcontent($this->ch); + $results = array(); $xpath = get_xpath($response); - $results = array(); + if (!$xpath) + return $results; foreach($xpath->query("//ol[@class='searchResults']//li[@class='result']") as $result) { diff --git a/engines/bittorrent/1337x.php b/engines/bittorrent/1337x.php index c70468f..f5390ec 100644 --- a/engines/bittorrent/1337x.php +++ b/engines/bittorrent/1337x.php @@ -11,6 +11,9 @@ $xpath = get_xpath($response); $results = array(); + if (!$xpath) + return $results; + foreach($xpath->query("//table/tbody/tr") as $result) { $name = $xpath->evaluate(".//td[@class='coll-1 name']/a", $result)[1]->textContent; $magnet = "./engines/bittorrent/get_magnet_1337x.php?url=https://1337x.to" . $xpath->evaluate(".//td[@class='coll-1 name']/a/@href", $result)[1]->textContent; diff --git a/engines/bittorrent/nyaa.php b/engines/bittorrent/nyaa.php index c250f1a..d3cc22a 100644 --- a/engines/bittorrent/nyaa.php +++ b/engines/bittorrent/nyaa.php @@ -11,6 +11,9 @@ $xpath = get_xpath($response); $results = array(); + if (!$xpath) + return $results; + foreach($xpath->query("//tbody/tr") as $result) { $name_node = $xpath->evaluate(".//td[@colspan='2']//a[not(contains(@class, 'comments'))]/@title", $result); diff --git a/engines/bittorrent/rutor.php b/engines/bittorrent/rutor.php index f6790a7..e575bac 100644 --- a/engines/bittorrent/rutor.php +++ b/engines/bittorrent/rutor.php @@ -9,6 +9,8 @@ $xpath = get_xpath($response); $results = array(); + if (!$xpath) + return $results; foreach($xpath->query("//table/tr[@class='gai' or @class='tum']") as $result) { diff --git a/engines/bittorrent/torrentgalaxy.php b/engines/bittorrent/torrentgalaxy.php index d893db3..2f6c606 100644 --- a/engines/bittorrent/torrentgalaxy.php +++ b/engines/bittorrent/torrentgalaxy.php @@ -6,33 +6,36 @@ } public function get_results() { - $response = curl_multi_getcontent($this->ch); - $xpath = get_xpath($response); - $results = array(); - - foreach($xpath->query("//div[@class='tgxtablerow txlight']") as $result) - { - $name = $xpath->evaluate(".//div[contains(@class, 'clickable-row')]", $result)[0]->textContent; - $magnet = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/a/@href", $result)[1]->textContent; - $magnet_without_tracker = explode("&tr=", $magnet)[0]; - $magnet = $magnet_without_tracker . $this->opts->bittorrent_trackers; - $size = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span", $result)[0]->textContent; - $seeders = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[1]->textContent; - $leechers = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[2]->textContent; - - array_push($results, - array ( - "name" => htmlspecialchars($name), - "seeders" => (int) $seeders, - "leechers" => (int) $leechers, - "magnet" => htmlspecialchars($magnet), - "size" => htmlspecialchars($size), - "source" => "torrentgalaxy.to" - ) - ); - } + $response = curl_multi_getcontent($this->ch); + $xpath = get_xpath($response); + $results = array(); + if (!$xpath) return $results; + + foreach($xpath->query("//div[@class='tgxtablerow txlight']") as $result) + { + $name = $xpath->evaluate(".//div[contains(@class, 'clickable-row')]", $result)[0]->textContent; + $magnet = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/a/@href", $result)[1]->textContent; + $magnet_without_tracker = explode("&tr=", $magnet)[0]; + $magnet = $magnet_without_tracker . $this->opts->bittorrent_trackers; + $size = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span", $result)[0]->textContent; + $seeders = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[1]->textContent; + $leechers = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[2]->textContent; + + array_push($results, + array ( + "name" => htmlspecialchars($name), + "seeders" => (int) $seeders, + "leechers" => (int) $leechers, + "magnet" => htmlspecialchars($magnet), + "size" => htmlspecialchars($size), + "source" => "torrentgalaxy.to" + ) + ); } + + return $results; } + } ?> diff --git a/engines/invidious/video.php b/engines/invidious/video.php index 7fefbf6..ad46a9a 100644 --- a/engines/invidious/video.php +++ b/engines/invidious/video.php @@ -1,7 +1,7 @@ instance_url = $this->config->invidious_instance_for_video_results; + $this->instance_url = $this->opts->invidious_instance_for_video_results; $query = urlencode($this->query); return "$this->instance_url/api/v1/search?q=$query"; } diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index 8492bc2..7d5e6a1 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -1,25 +1,28 @@ instance = $instance; - parent::__construct($opts, $mh); + class LibreXFallback extends EngineRequest { + public function __construct($instance, $opts, $mh) { + $this->instance = $instance; + parent::__construct($opts, $mh); + } + + public function get_request_url() { + return $this->instance . "api.php?" . opts_to_params($this->opts); + } + + public function get_results() { + $response = curl_exec($this->ch); + $response = json_decode($response, true); + if (!$response) + return array(); + + return array_values($response); + } } - public function get_request_url() { - return $instance . "api.php?" .opts_to_params($opts); - } -} - - - function get_librex_results($opts, $mh) { - global $config; - - if (isset($_REQUEST["nfb"]) && $_REQUEST["nfb"] == "1") - return array(); - - if (!$config->instance_fallback) + function get_librex_results($opts) { + if (!$opts->do_fallback) return array(); $instances_json = json_decode(file_get_contents("instances.json"), true); @@ -27,12 +30,11 @@ class LibreXFallback extends EngineRequest { if (empty($instances_json["instances"])) return array(); + // TODO pick instances which aren't on cooldown $instances = array_map(fn($n) => $n['clearnet'], array_filter($instances_json['instances'], fn($n) => !is_null($n['clearnet']))); shuffle($instances); - $query_encoded = urlencode($query); - $results = array(); $tries = 0; @@ -44,18 +46,13 @@ class LibreXFallback extends EngineRequest { if (parse_url($instance)["host"] == parse_url($_SERVER['HTTP_HOST'])["host"]) continue; - $url = $instance . "api.php?q=$query_encoded&p=$page&t=0&nfb=1"; + $librex_request = new LibreXFallback($instance, $opts, null); + $results = $librex_request->get_results(); - $librex_ch = curl_init($url); - curl_setopt_array($librex_ch, $config->curl_settings); - copy_cookies($librex_ch); - $response = curl_exec($librex_ch); - curl_close($librex_ch); + if (count($results) > 1) + return $results; - $code = curl_getinfo($librex_ch)["http_code"]; - $results = json_decode($response, true); - - } while ( !empty($instances) && ($results == null || count($results) <= 1)); + } while ( !empty($instances)); if (empty($instances)) return array(); diff --git a/engines/qwant/image.php b/engines/qwant/image.php index a32dfce..2655008 100644 --- a/engines/qwant/image.php +++ b/engines/qwant/image.php @@ -11,6 +11,9 @@ $results = array(); $xpath = get_xpath(curl_multi_getcontent($this->ch)); + if (!$xpath) + return $results; + foreach($xpath->query("//a[@rel='noopener']") as $result) { $image = $xpath->evaluate(".//img", $result)[0]; diff --git a/engines/text/duckduckgo.php b/engines/text/duckduckgo.php index 98026c9..7bcff56 100644 --- a/engines/text/duckduckgo.php +++ b/engines/text/duckduckgo.php @@ -57,7 +57,7 @@ ) ); } - return $results; + return $results; } } diff --git a/engines/text/google.php b/engines/text/google.php index 6104d48..c12f9fa 100644 --- a/engines/text/google.php +++ b/engines/text/google.php @@ -51,7 +51,7 @@ } $url = $url->textContent; - $url = check_for_privacy_frontend($url, $opts); + $url = check_for_privacy_frontend($url, $this->opts); $title = $xpath->evaluate(".//h3", $result)[0]; $description = $xpath->evaluate(".//div[contains(@class, 'VwiC3b')]", $result)[0]; diff --git a/engines/text/text.php b/engines/text/text.php index df29438..cdaaa4a 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -27,11 +27,8 @@ } public function get_results() { - if (curl_getinfo($this->engine_request->ch)['http_code'] != '200') - { - require "engines/librex/text.php"; - return get_librex_results($this->query, $this->page); - } + if (curl_getinfo($this->engine_request->ch)['http_code'] != '200') + return array(); $results = $this->engine_request->get_results(); diff --git a/misc/search_engine.php b/misc/search_engine.php index 3561a0a..6b38fe4 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -6,17 +6,21 @@ $this->opts = $opts; $url = $this->get_request_url(); + error_log($url); if ($url) { $this->ch = curl_init($url); - curl_setopt_array($this->ch, $opts->curl_settings); - curl_multi_add_handle($mh, $this->ch); + + if ($opts->curl_settings) + curl_setopt_array($this->ch, $opts->curl_settings); + + if ($mh) + curl_multi_add_handle($mh, $this->ch); } } public function get_request_url(){ return ""; } - public function successful() { return curl_getinfo($this->ch)['http_code'] == '200'; } @@ -41,7 +45,7 @@ $opts->disable_frontends = (int) ($_REQUEST["nf"] ?? 0) == 1 || isset($_COOKIE["disable_frontends"]); - $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"])); + $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? "")); $opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0; if (!$opts->instance_fallback) { @@ -63,10 +67,10 @@ $params .= "p=$opts->page"; $params .= "&q=$query"; $params .= "&t=$opts->type"; - $params .= "&nfb=" . $opts->do_fallback ? 0 : 1; - $params .= "&safe=" . $opts->safe_search ? 1 : 0; - $params .= "&nf=" . $opts->disable_frontends ? 1 : 0; - $params .= "&ns=" . $opts->disable_special ? 1 : 0; + $params .= "&nfb=" . ($opts->do_fallback ? 0 : 1); + $params .= "&safe=" . ($opts->safe_search ? 1 : 0); + $params .= "&nf=" . ($opts->disable_frontends ? 1 : 0); + $params .= "&ns=" . ($opts->disable_special ? 1 : 0); return $params; } @@ -119,8 +123,8 @@ $results = $search_category->get_results(); if (empty($results)) { - //if (!$opt->do_fallback) - //return ; + require "engines/librex/fallback.php"; + $results = get_librex_results($opts); } if (!$do_print) diff --git a/misc/tools.php b/misc/tools.php index 23d41bf..ee2292d 100644 --- a/misc/tools.php +++ b/misc/tools.php @@ -17,14 +17,12 @@ return $root_domain; } - function try_replace_with_frontend($url, $frontend, $original) - { - global $config; + function try_replace_with_frontend($url, $frontend, $original, $opts) { $frontends = $opts->frontends; - $frontend = $opts->$frontend["instance_url"]; + $frontend = $opts->$frontend; if ($frontend) { - + $frontend = $frontend->instance_url; if (empty(trim($frontend))) return $url; @@ -67,10 +65,10 @@ $original = $data["original_url"]; if (strpos($url, $original)) { - $url = try_replace_with_frontend($url, $frontend, $original); + $url = try_replace_with_frontend($url, $frontend, $original, $opts); break; } else if (strpos($url, "stackexchange.com")) { - $url = try_replace_with_frontend($url, "anonymousoverflow", "stackexchange.com"); + $url = try_replace_with_frontend($url, "anonymousoverflow", "stackexchange.com", $opts); break; } } @@ -79,6 +77,9 @@ } function get_xpath($response) { + if (!$response) + return null; + $htmlDom = new DOMDocument; @$htmlDom->loadHTML($response); $xpath = new DOMXPath($htmlDom); From 14cdccb83f2ac83c91464f90111b61f51d71d164 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 23 Aug 2023 00:51:23 +0100 Subject: [PATCH 14/40] add theme to query parameter --- misc/header.php | 3 ++- misc/search_engine.php | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/misc/header.php b/misc/header.php index 2d6db46..b7944bf 100644 --- a/misc/header.php +++ b/misc/header.php @@ -8,5 +8,6 @@ "/> diff --git a/misc/search_engine.php b/misc/search_engine.php index 6b38fe4..9b42f14 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -37,7 +37,7 @@ $opts->page = (int) ($_REQUEST["p"] ?? 0); $opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0; - $opts->theme = trim(htmlspecialchars($_COOKIE["theme"] ?? "dark")); + $opts->theme = $_REQUEST["theme"] ?? trim(htmlspecialchars($_COOKIE["theme"] ?? "dark")); $opts->safe_search = (int) ($_REQUEST["safe"] ?? 0) == 1 || isset($_COOKIE["safe_search"]); From 1e150060678104010d0c5cd72caa1f0a475fb7fb Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 23 Aug 2023 00:54:10 +0100 Subject: [PATCH 15/40] Remove duplicate nfb parameter --- misc/search_engine.php | 1 - 1 file changed, 1 deletion(-) diff --git a/misc/search_engine.php b/misc/search_engine.php index 9b42f14..5d4c256 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -35,7 +35,6 @@ $opts->query = trim($_REQUEST["q"] ?? ""); $opts->type = (int) ($_REQUEST["t"] ?? 0); $opts->page = (int) ($_REQUEST["p"] ?? 0); - $opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0; $opts->theme = $_REQUEST["theme"] ?? trim(htmlspecialchars($_COOKIE["theme"] ?? "dark")); From 8ede57f4f46d46521b77b1f940bc537029f80843 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 23 Aug 2023 02:32:47 +0100 Subject: [PATCH 16/40] fix issue with not actually using duckduckgo --- engines/text/duckduckgo.php | 2 +- engines/text/google.php | 7 ++++--- engines/text/text.php | 5 +---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/engines/text/duckduckgo.php b/engines/text/duckduckgo.php index 7bcff56..378ad4e 100644 --- a/engines/text/duckduckgo.php +++ b/engines/text/duckduckgo.php @@ -41,7 +41,7 @@ $url = $url->textContent; - $url = check_for_privacy_frontend($url, $opts); + $url = check_for_privacy_frontend($url, $this->opts); $title = $xpath->evaluate(".//h2[@class='result__title']", $result)[0]; $description = $xpath->evaluate(".//a[@class='result__snippet']", $result)[0]; diff --git a/engines/text/google.php b/engines/text/google.php index c12f9fa..4aa2523 100644 --- a/engines/text/google.php +++ b/engines/text/google.php @@ -25,11 +25,13 @@ } - public function get_results() { $results = array(); $xpath = get_xpath(curl_multi_getcontent($this->ch)); + if (!$xpath) + return $results; + $didyoumean = $xpath->query(".//a[@class='gL9Hy']")[0]; if (!is_null($didyoumean)) @@ -37,8 +39,7 @@ "did_you_mean" => $didyoumean->textContent )); - foreach($xpath->query("//div[@id='search']//div[contains(@class, 'g')]") as $result) - { + foreach($xpath->query("//div[@id='search']//div[contains(@class, 'g')]") as $result) { $url = $xpath->evaluate(".//div[@class='yuRUbf']//a/@href", $result)[0]; if ($url == null) diff --git a/engines/text/text.php b/engines/text/text.php index cdaaa4a..fb4a751 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -5,7 +5,7 @@ $this->page = $opts->page; $this->opts = $opts; - $engine = $opts->preferred_engines->text ?? "google"; + $engine = $opts->preferred_engines["text"] ?? "google"; $query_parts = explode(" ", $this->query); $last_word_query = end($query_parts); @@ -27,9 +27,6 @@ } public function get_results() { - if (curl_getinfo($this->engine_request->ch)['http_code'] != '200') - return array(); - $results = $this->engine_request->get_results(); if ($this->special_request) { From 1b694e088243b5fd9cc5a0232c0cc227ecb7a569 Mon Sep 17 00:00:00 2001 From: davidovski Date: Wed, 23 Aug 2023 10:55:18 +0100 Subject: [PATCH 17/40] Fix frontends being parsed from config --- misc/search_engine.php | 4 +++- misc/tools.php | 13 +++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/misc/search_engine.php b/misc/search_engine.php index 5d4c256..b22941d 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -21,6 +21,7 @@ public function get_request_url(){ return ""; } + public function successful() { return curl_getinfo($this->ch)['http_code'] == '200'; } @@ -54,7 +55,8 @@ $opts->number_of_results ??= trim(htmlspecialchars($_COOKIE["number_of_results"])); foreach (array_keys($opts->frontends ?? array()) as $frontend) { - $opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? ""; + if (!$opts->frontends[$frontend]["instance_url"]) + $opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? ""; } return $opts; } diff --git a/misc/tools.php b/misc/tools.php index ee2292d..20ffa9b 100644 --- a/misc/tools.php +++ b/misc/tools.php @@ -1,28 +1,25 @@ frontends; - $frontend = $opts->$frontend; - if ($frontend) { - $frontend = $frontend->instance_url; + if (array_key_exists($frontend, $opts->frontends)) { + $frontend = $frontends[$frontend]["instance_url"]; if (empty(trim($frontend))) return $url; From 5ce0f2ccd02a01423a4b001c8437a7de79ec08e2 Mon Sep 17 00:00:00 2001 From: davidovski Date: Thu, 24 Aug 2023 03:57:11 +0100 Subject: [PATCH 18/40] fix video and image search --- engines/invidious/video.php | 11 ++++------- engines/qwant/image.php | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/engines/invidious/video.php b/engines/invidious/video.php index ad46a9a..17264df 100644 --- a/engines/invidious/video.php +++ b/engines/invidious/video.php @@ -11,13 +11,11 @@ $response = curl_multi_getcontent($this->ch); $json_response = json_decode($response, true); - foreach ($json_response as $response) - { - if ($response["type"] == "video") - { + foreach ($json_response as $response) { + if ($response["type"] == "video") { $title = $response["title"]; $url = "https://youtube.com/watch?v=" . $response["videoId"]; - $url = check_for_privacy_frontend($url); + $url = check_for_privacy_frontend($url, $this-opts); $uploader = $response["author"]; $views = $response["viewCount"]; $date = $response["publishedText"]; @@ -43,8 +41,7 @@ public static function print_results($results) { echo "
"; - foreach($results as $result) - { + foreach($results as $result) { $title = $result["title"]; $url = $result["url"]; $base_url = $result["base_url"]; diff --git a/engines/qwant/image.php b/engines/qwant/image.php index 2655008..7459b18 100644 --- a/engines/qwant/image.php +++ b/engines/qwant/image.php @@ -24,7 +24,7 @@ $encoded_url_split1 = explode("==/", $encoded_url)[1]; $encoded_url_split2 = explode("?position", $encoded_url_split1)[0]; $real_url = urldecode(base64_decode($encoded_url_split2)); - $real_url = check_for_privacy_frontend($real_url); + $real_url = check_for_privacy_frontend($real_url, $this->opts); $alt = $image->getAttribute("alt"); $thumbnail = urlencode($image->getAttribute("src")); From 508ac0c576adc5314c3b8d6e5287a5ee2506b461 Mon Sep 17 00:00:00 2001 From: davidovski Date: Thu, 24 Aug 2023 15:45:45 +0100 Subject: [PATCH 19/40] Use cookie frontend over config frontend --- engines/invidious/video.php | 2 +- misc/search_engine.php | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/engines/invidious/video.php b/engines/invidious/video.php index 17264df..afd51b2 100644 --- a/engines/invidious/video.php +++ b/engines/invidious/video.php @@ -15,7 +15,7 @@ if ($response["type"] == "video") { $title = $response["title"]; $url = "https://youtube.com/watch?v=" . $response["videoId"]; - $url = check_for_privacy_frontend($url, $this-opts); + $url = check_for_privacy_frontend($url, $this->opts); $uploader = $response["author"]; $views = $response["viewCount"]; $date = $response["publishedText"]; diff --git a/misc/search_engine.php b/misc/search_engine.php index b22941d..5ef99c4 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -6,19 +6,19 @@ $this->opts = $opts; $url = $this->get_request_url(); - error_log($url); - if ($url) { - $this->ch = curl_init($url); + if (!$url) + return; - if ($opts->curl_settings) - curl_setopt_array($this->ch, $opts->curl_settings); + $this->ch = curl_init($url); - if ($mh) - curl_multi_add_handle($mh, $this->ch); - } + if ($opts->curl_settings) + curl_setopt_array($this->ch, $opts->curl_settings); + + if ($mh) + curl_multi_add_handle($mh, $this->ch); } - public function get_request_url(){ + public function get_request_url() { return ""; } @@ -55,8 +55,7 @@ $opts->number_of_results ??= trim(htmlspecialchars($_COOKIE["number_of_results"])); foreach (array_keys($opts->frontends ?? array()) as $frontend) { - if (!$opts->frontends[$frontend]["instance_url"]) - $opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? ""; + $opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? $opts->frontends[$frontend]["instance_url"]; } return $opts; } From 036b10efe7f6bd8219247d116a08253fa25c7c52 Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 00:03:12 +0100 Subject: [PATCH 20/40] Add hl back into google query --- engines/text/google.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/engines/text/google.php b/engines/text/google.php index 4aa2523..d43ff1d 100644 --- a/engines/text/google.php +++ b/engines/text/google.php @@ -12,8 +12,10 @@ $url = "https://www.google.$domain/search?q=$query_encoded&nfpr=1&start=$this->page"; - if (3 > strlen($results_language) && 0 < strlen($results_language)) + if (3 > strlen($results_language) && 0 < strlen($results_language)) { $url .= "&lr=lang_$results_language"; + $url .= "&hl=$results_language"; + } if (3 > strlen($number_of_results) && 0 < strlen($number_of_results)) $url .= "&num=$number_of_results"; From 04e0183d79daaef3c19cc116f4e9736511ba5b90 Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 00:10:03 +0100 Subject: [PATCH 21/40] Fix missing arg error --- engines/text/text.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engines/text/text.php b/engines/text/text.php index fb4a751..ea983be 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -10,7 +10,7 @@ $query_parts = explode(" ", $this->query); $last_word_query = end($query_parts); if (substr($this->query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!") - check_ddg_bang($this->query); + check_ddg_bang($this->query, $opts); if ($engine == "google") { require "engines/text/google.php"; @@ -97,7 +97,7 @@ } } - function check_ddg_bang($query) + function check_ddg_bang($query, $opts) { $bangs_json = file_get_contents("static/misc/ddg_bang.json"); @@ -125,7 +125,7 @@ $bang_query = trim(implode("", $bang_query_array)); $request_url = str_replace("{{{s}}}", str_replace('%26quot%3B','%22', urlencode($bang_query)), $bang_url); - $request_url = check_for_privacy_frontend($request_url); + $request_url = check_for_privacy_frontend($request_url, $opts); header("Location: " . $request_url); die(); From 723887bfc0f3d37d59fb9e3cf0918b6c5cbb8127 Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 00:12:19 +0100 Subject: [PATCH 22/40] Fix typo in variable name for instances --- instances.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instances.php b/instances.php index dc7c9b2..8e26a8f 100644 --- a/instances.php +++ b/instances.php @@ -19,7 +19,7 @@ foreach($instances as $instance) { $hostname = parse_url($instance["clearnet"])["host"]; - $country = get_country_emote($instance["country"]) . $instnace["country"]; + $country = get_country_emote($instance["country"]) . $instance["country"]; $is_tor = !is_null($instance["tor"]); $is_i2p = !is_null($instance["i2p"]); From a7d1836eff46f789c88811f6506c4b7fcafd8d3d Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 11:36:19 +0100 Subject: [PATCH 23/40] Fixed fallback not occuring when special result --- config.php.example | 33 +++++++++++++++++---------------- misc/search_engine.php | 2 +- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/config.php.example b/config.php.example index 9d28718..f906c72 100644 --- a/config.php.example +++ b/config.php.example @@ -17,7 +17,8 @@ "disable_hidden_service_search" => false, // Fallback to another librex instance if google search fails - "instance_fallback" => false, // This might generate a 504 Gateway Timeout error, we are looking into this. + // This may greatly increase the time it takes to get a result and in some cases results in 504 errors + "instance_fallback" => false, /* Preset privacy friendly frontends for users, these can be overwritten by users in the settings @@ -27,74 +28,74 @@ "frontends" => array( "invidious" => array( "instance_url" => "", - "project_url" => "https://docs.invidious.io/instances/", + "project_url" => "https://docs.invidious.io/instances/", "original_name" => "YouTube", "original_url" => "youtube.com" ), "rimgo" => array( "instance_url" => "", - "project_url" => "https://codeberg.org/video-prize-ranch/rimgo#instances", + "project_url" => "https://codeberg.org/video-prize-ranch/rimgo#instances", "original_name" => "Imgur", "original_url" => "imgur.com" ), "scribe" => array( "instance_url" => "", - "project_url" => "https://git.sr.ht/~edwardloveall/scribe/tree/main/docs/instances.md", + "project_url" => "https://git.sr.ht/~edwardloveall/scribe/tree/main/docs/instances.md", "original_name" => "Medium", "original_url" => "medium.com" ), "gothub" => array( "instance_url" => "", - "project_url" => "https://codeberg.org/gothub/gothub#instances", + "project_url" => "https://codeberg.org/gothub/gothub#instances", "original_name" => "GitHub", "original_url" => "github.com" ), "nitter" => array( "instance_url" => "", - "project_url" => "https://github.com/zedeus/nitter/wiki/Instances", + "project_url" => "https://github.com/zedeus/nitter/wiki/Instances", "original_name" => "Twitter", "original_url" => "twitter.com" ), "libreddit" => array( "instance_url" => "", - "project_url" => "https://github.com/libreddit/libreddit-instances/blob/master/instances.md", + "project_url" => "https://github.com/libreddit/libreddit-instances/blob/master/instances.md", "original_name" => "Reddit", "original_url" => "reddit.com" ), "proxitok" => array( "instance_url" => "", - "project_url" => "https://github.com/pablouser1/ProxiTok/wiki/Public-instances", + "project_url" => "https://github.com/pablouser1/ProxiTok/wiki/Public-instances", "original_name" => "TikTok", "original_url" => "tiktok.com" ), "wikiless" => array( "instance_url" => "", - "project_url" => "https://github.com/Metastem/wikiless#instances", + "project_url" => "https://github.com/Metastem/wikiless#instances", "original_name" => "Wikipedia", "original_url" => "wikipedia.org" ), "quetre" => array( "instance_url" => "", - "project_url" => "https://github.com/zyachel/quetre#instances", + "project_url" => "https://github.com/zyachel/quetre#instances", "original_name" => "Quora", "original_url" => "quora.com" ), "libremdb" => array( "instance_url" => "", - "project_url" => "https://github.com/zyachel/libremdb#instances", + "project_url" => "https://github.com/zyachel/libremdb#instances", "original_name" => "IMDb", "original_url" => "imdb.com" ), "breezewiki" => array( "instance_url" => "", - "project_url" => "https://docs.breezewiki.com/Links.html", + "project_url" => "https://docs.breezewiki.com/Links.html", "original_name" => "Fandom", "original_url" => "fandom.com" ), "anonymousoverflow" => array( "instance_url" => "", - "project_url" => "https://github.com/httpjamesm/AnonymousOverflow#clearnet-instances", + "project_url" => "https://github.com/httpjamesm/AnonymousOverflow#clearnet-instances", "original_name" => "StackOverflow", "original_url" => "stackoverflow.com" ), @@ -111,10 +112,10 @@ "original_url" => "goodreads.com" ) ), - + "preferred_engines" => array( - + /* replace with "text" => "duckduckgo" to use duckduckgo instead * (recommended if being ratelimited */ "text" => "google" @@ -146,7 +147,7 @@ CURLOPT_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, CURLOPT_MAXREDIRS => 5, - CURLOPT_TIMEOUT => 18, + CURLOPT_TIMEOUT => 3, CURLOPT_VERBOSE => false ) ); diff --git a/misc/search_engine.php b/misc/search_engine.php index 5ef99c4..1c1c100 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -122,7 +122,7 @@ $results = $search_category->get_results(); - if (empty($results)) { + if (count($results) <= 1) { require "engines/librex/fallback.php"; $results = get_librex_results($opts); } From 0fd041a35372a0b5aed12be2be2097bddb66982b Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 16:23:53 +0100 Subject: [PATCH 24/40] Add timeouts for failing instances --- config.php.example | 3 +++ engines/librex/fallback.php | 39 +++++++++++++++++++++++++------------ misc/cooldowns.php | 20 +++++++++++++++++++ misc/search_engine.php | 1 + 4 files changed, 51 insertions(+), 12 deletions(-) create mode 100644 misc/cooldowns.php diff --git a/config.php.example b/config.php.example index f906c72..599278b 100644 --- a/config.php.example +++ b/config.php.example @@ -20,6 +20,9 @@ // This may greatly increase the time it takes to get a result and in some cases results in 504 errors "instance_fallback" => false, + // how long in minutes to put google/other instances on cooldown if they aren't responding + "request_cooldown" => 5, + /* Preset privacy friendly frontends for users, these can be overwritten by users in the settings e.g.: Preset the invidious instance URL: "instance_url" => "https://yewtu.be", diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index 7d5e6a1..5244c38 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -15,25 +15,38 @@ $response = json_decode($response, true); if (!$response) return array(); - + return array_values($response); } } - - function get_librex_results($opts) { - if (!$opts->do_fallback) - return array(); - + function load_instances($cooldowns) { $instances_json = json_decode(file_get_contents("instances.json"), true); if (empty($instances_json["instances"])) return array(); - // TODO pick instances which aren't on cooldown - $instances = array_map(fn($n) => $n['clearnet'], array_filter($instances_json['instances'], fn($n) => !is_null($n['clearnet']))); + $instances = array_filter($instances, fn($n) => !has_cooldown($n, $cooldowns)); shuffle($instances); + return $instances; + } + + function get_librex_results($opts) { + if (!$opts->do_fallback) + return array(); + + require "misc/cooldowns.php"; + + $cooldowns = load_cooldowns(); + error_log("loaded" . count($cooldowns)); + error_log(print_r($cooldowns,true)); + + echo "
";
+        print_r($cooldowns);
+        echo "
"; + + $instances = load_instances($cooldowns); $results = array(); $tries = 0; @@ -43,6 +56,7 @@ $instance = array_pop($instances); + error_log($instance . "--- " . parse_url($instance)["host"]); if (parse_url($instance)["host"] == parse_url($_SERVER['HTTP_HOST'])["host"]) continue; @@ -52,12 +66,13 @@ if (count($results) > 1) return $results; - } while ( !empty($instances)); + // on fail then do this + $timeout = ($opts->request_cooldown ?? "1") * 60; + $cooldowns = set_cooldown($instance, $timeout, $cooldowns); - if (empty($instances)) - return array(); + } while (!empty($instances)); - return array_values($results); + return array(); } ?> diff --git a/misc/cooldowns.php b/misc/cooldowns.php new file mode 100644 index 0000000..975afc0 --- /dev/null +++ b/misc/cooldowns.php @@ -0,0 +1,20 @@ + time(); + } +?> diff --git a/misc/search_engine.php b/misc/search_engine.php index 1c1c100..0b2e221 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -57,6 +57,7 @@ foreach (array_keys($opts->frontends ?? array()) as $frontend) { $opts->frontends[$frontend]["instance_url"] = $_COOKIE[$frontend] ?? $opts->frontends[$frontend]["instance_url"]; } + return $opts; } From f50022f7807683aaaa4e72dd78123e575baf6a6c Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 17:39:08 +0100 Subject: [PATCH 25/40] added request cooldowns for google --- config.php.example | 3 ++- engines/librex/fallback.php | 4 +--- engines/text/google.php | 3 +-- engines/text/text.php | 17 ++++++++++++++--- misc/search_engine.php | 3 +++ 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/config.php.example b/config.php.example index 599278b..b69c1dc 100644 --- a/config.php.example +++ b/config.php.example @@ -151,7 +151,8 @@ CURLOPT_REDIR_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, CURLOPT_MAXREDIRS => 5, CURLOPT_TIMEOUT => 3, - CURLOPT_VERBOSE => false + CURLOPT_VERBOSE => false, + CURLOPT_FOLLOWLOCATION => true ) ); ?> diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index 5244c38..a56c7bb 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -36,9 +36,7 @@ if (!$opts->do_fallback) return array(); - require "misc/cooldowns.php"; - - $cooldowns = load_cooldowns(); + $cooldowns = $opts->cooldowns; error_log("loaded" . count($cooldowns)); error_log(print_r($cooldowns,true)); diff --git a/engines/text/google.php b/engines/text/google.php index d43ff1d..45c871d 100644 --- a/engines/text/google.php +++ b/engines/text/google.php @@ -1,7 +1,6 @@ query)); $results = array(); diff --git a/engines/text/text.php b/engines/text/text.php index ea983be..2122d65 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -5,19 +5,23 @@ $this->page = $opts->page; $this->opts = $opts; - $engine = $opts->preferred_engines["text"] ?? "google"; + $this->engine = $opts->preferred_engines["text"] ?? "google"; $query_parts = explode(" ", $this->query); $last_word_query = end($query_parts); if (substr($this->query, 0, 1) == "!" || substr($last_word_query, 0, 1) == "!") check_ddg_bang($this->query, $opts); - if ($engine == "google") { + if (has_cooldown($this->engine, $this->opts->cooldowns)) + return; + + if ($this->engine == "google") { + require "engines/text/google.php"; $this->engine_request = new GoogleRequest($opts, $mh); } - if ($engine == "duckduckgo") { + if ($this->engine == "duckduckgo") { require "engines/text/duckduckgo.php"; $this->engine_request = new DuckDuckGoRequest($opts, $mh); } @@ -27,6 +31,10 @@ } public function get_results() { + if (!$this->engine_request) + return array(); + + error_log("fetching googl results"); $results = $this->engine_request->get_results(); if ($this->special_request) { @@ -36,6 +44,9 @@ $results = array_merge(array($special_result), $results); } + if (count($results) <= 1) + set_cooldown($this->engine, ($opts->request_cooldown ?? "1") * 60, $this->opts->cooldowns); + return $results; } diff --git a/misc/search_engine.php b/misc/search_engine.php index 0b2e221..89f9176 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -111,6 +111,9 @@ } function fetch_search_results($opts, $do_print) { + require "misc/cooldowns.php"; + $opts->cooldowns = load_cooldowns(); + $start_time = microtime(true); $mh = curl_multi_init(); $search_category = init_search($opts, $mh); From 039e549d4ce11143521b7909bcb563b9ea178cc3 Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 17:48:45 +0100 Subject: [PATCH 26/40] Change default timeout to 25 minutes --- config.php.example | 6 +++--- engines/librex/fallback.php | 8 -------- engines/text/text.php | 1 - misc/cooldowns.php | 1 - 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/config.php.example b/config.php.example index b69c1dc..567e839 100644 --- a/config.php.example +++ b/config.php.example @@ -17,11 +17,11 @@ "disable_hidden_service_search" => false, // Fallback to another librex instance if google search fails - // This may greatly increase the time it takes to get a result and in some cases results in 504 errors - "instance_fallback" => false, + // This may greatly increase the time it takes to get a result, if a direct search is not possible + "instance_fallback" => true, // how long in minutes to put google/other instances on cooldown if they aren't responding - "request_cooldown" => 5, + "request_cooldown" => 25, /* Preset privacy friendly frontends for users, these can be overwritten by users in the settings diff --git a/engines/librex/fallback.php b/engines/librex/fallback.php index a56c7bb..4ff13b0 100644 --- a/engines/librex/fallback.php +++ b/engines/librex/fallback.php @@ -37,13 +37,6 @@ return array(); $cooldowns = $opts->cooldowns; - error_log("loaded" . count($cooldowns)); - error_log(print_r($cooldowns,true)); - - echo "
";
-        print_r($cooldowns);
-        echo "
"; - $instances = load_instances($cooldowns); $results = array(); @@ -54,7 +47,6 @@ $instance = array_pop($instances); - error_log($instance . "--- " . parse_url($instance)["host"]); if (parse_url($instance)["host"] == parse_url($_SERVER['HTTP_HOST'])["host"]) continue; diff --git a/engines/text/text.php b/engines/text/text.php index 2122d65..26af3ae 100644 --- a/engines/text/text.php +++ b/engines/text/text.php @@ -34,7 +34,6 @@ if (!$this->engine_request) return array(); - error_log("fetching googl results"); $results = $this->engine_request->get_results(); if ($this->special_request) { diff --git a/misc/cooldowns.php b/misc/cooldowns.php index 975afc0..70d0bbe 100644 --- a/misc/cooldowns.php +++ b/misc/cooldowns.php @@ -9,7 +9,6 @@ function set_cooldown($instance, $timeout, $cooldowns) { $cooldowns[$instance] = time() + $timeout; - error_log("cooldown on instnace $instance for $timeout seconds"); save_cooldowns($cooldowns); return $cooldowns; } From b8047b298247d061d8e34bc58d2d5b1373f98936 Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 17:53:59 +0100 Subject: [PATCH 27/40] add default for request_cooldown --- misc/search_engine.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misc/search_engine.php b/misc/search_engine.php index 89f9176..75e87a5 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -33,6 +33,8 @@ function load_opts() { $opts = require "config.php"; + $opts->request_cooldown ??= 25; + $opts->query = trim($_REQUEST["q"] ?? ""); $opts->type = (int) ($_REQUEST["t"] ?? 0); $opts->page = (int) ($_REQUEST["p"] ?? 0); From 8bae58c2b28dcc3f9d1eec2291e09a4a76070b85 Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 18:06:12 +0100 Subject: [PATCH 28/40] Add php8-apcu to dependencies --- README.md | 2 +- docker/php/php.dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f56505a..44e781a 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ You can access the full list of LibreX and LibreY instances on one of the follow ### About LibreY LibreY gives you text results from DuckDuckGo or Google, images from Qwant, and torrents from i.e. Ahmia and popular torrent sites without spying on you. -
LibreY doesn't save any type of data about the user, there are no logs (except NGINX logs if the host sets them), no caches. +
LibreY doesn't save **any** type of data about the user, there are no logs (except NGINX logs if the host sets them). ### LibreY compared to other metasearch engines diff --git a/docker/php/php.dockerfile b/docker/php/php.dockerfile index 5a52265..8735137 100644 --- a/docker/php/php.dockerfile +++ b/docker/php/php.dockerfile @@ -56,7 +56,7 @@ ENV CURLOPT_VERBOSE=true # Install PHP-FPM using Alpine's package manager, apk # Configure PHP-FPM to listen on a Unix socket instead of a TCP port, which is more secure and efficient -RUN apk add php8 php8-fpm php8-dom php8-curl php8-json --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing &&\ +RUN apk add php8 php8-fpm php8-dom php8-curl php8-json php8-apcu --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing &&\ sed -i 's/^\s*listen = 127.0.0.1:9000/listen = \/run\/php8\/php-fpm8.sock/' ${WWW_CONFIG} &&\ sed -i 's/^\s*;\s*listen.owner = nobody/listen.owner = nginx/' ${WWW_CONFIG} &&\ sed -i 's/^\s*;\s*listen.group = nobody/listen.group = nginx/' ${WWW_CONFIG} &&\ From 11e11192a47b91e22e0f5726a060920b8ede03b5 Mon Sep 17 00:00:00 2001 From: davidovski Date: Fri, 25 Aug 2023 18:57:23 +0100 Subject: [PATCH 29/40] Do not persist if apcu is not installed --- misc/cooldowns.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/misc/cooldowns.php b/misc/cooldowns.php index 70d0bbe..1cc0fff 100644 --- a/misc/cooldowns.php +++ b/misc/cooldowns.php @@ -1,10 +1,13 @@ Date: Fri, 25 Aug 2023 19:09:06 +0100 Subject: [PATCH 30/40] Fixed incorrect use of function_exists --- misc/cooldowns.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/cooldowns.php b/misc/cooldowns.php index 1cc0fff..0285f47 100644 --- a/misc/cooldowns.php +++ b/misc/cooldowns.php @@ -1,12 +1,12 @@ Date: Fri, 25 Aug 2023 23:15:17 +0100 Subject: [PATCH 31/40] Fix missing quote in docker build script --- docker/attributes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/attributes.sh b/docker/attributes.sh index 900fd9d..95a1c8c 100755 --- a/docker/attributes.sh +++ b/docker/attributes.sh @@ -23,7 +23,7 @@ export CONFIG_GOOGLE_DOMAIN="${CONFIG_GOOGLE_DOMAIN:-"com"}" export CONFIG_GOOGLE_LANGUAGE_SITE="${CONFIG_GOOGLE_LANGUAGE_SITE:-"en"}" export CONFIG_GOOGLE_LANGUAGE_RESULTS="${CONFIG_GOOGLE_LANGUAGE_RESULTS:-"en"}" export CONFIG_GOOGLE_NUMBER_OF_RESULTS="${CONFIG_GOOGLE_NUMBER_OF_RESULTS:-"10"}" -export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK}:-true} +export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK}:-true}" export CONFIG_INVIDIOUS_INSTANCE="${CONFIG_INVIDIOUS_INSTANCE:-"invidious.snopyta.org"}" export CONFIG_HIDDEN_SERVICE_SEARCH=${CONFIG_HIDDEN_SERVICE_SEARCH:-false} export CONFIG_DISABLE_BITTORRENT_SEARCH=${CONFIG_DISABLE_BITTORRENT_SEARCH:-false} From 98707900bfd6119a9a39183cda89de55939c44ec Mon Sep 17 00:00:00 2001 From: davidovski Date: Sat, 26 Aug 2023 00:41:25 +0100 Subject: [PATCH 32/40] Fix language config variable not working --- misc/search_engine.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/search_engine.php b/misc/search_engine.php index 75e87a5..b76cdcb 100644 --- a/misc/search_engine.php +++ b/misc/search_engine.php @@ -47,7 +47,7 @@ $opts->disable_frontends = (int) ($_REQUEST["nf"] ?? 0) == 1 || isset($_COOKIE["disable_frontends"]); - $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? "")); + $opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? $opts->language)); $opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0; if (!$opts->instance_fallback) { From 06ee3f06e8541e62dc5e0ba6d0864aea8b22359f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Dobo=C5=A1?= <26146502+kubo6472@users.noreply.github.com> Date: Sat, 26 Aug 2023 23:53:30 +0200 Subject: [PATCH 33/40] build docker image --- .github/workflows/docker-image.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/docker-image.yml diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 0000000..1427a4d --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,30 @@ +name: Docker Image CI +on: + push: + branches: [ "main" ] + workflow_dispatch: +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: docker/login-action@v2.1.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: owner + uses: ASzc/change-string-case-action@v5 + with: + string: ${{ github.repository_owner }} + - id: repo + uses: ASzc/change-string-case-action@v5 + with: + string: ${{ github.event.repository.name }} + - uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile + push: true + tags: | + ghcr.io/${{ steps.owner.outputs.lowercase }}/${{ steps.repo.outputs.lowercase }}:latest From e6c0bfda33dbc93742985fb1be005703b615f2b6 Mon Sep 17 00:00:00 2001 From: osarthak Date: Sun, 27 Aug 2023 15:55:17 +0530 Subject: [PATCH 34/40] fixed sub-search-button border padding, did-you-mean on mobile, smaller footer --- search.php | 1 - static/css/styles.css | 24 ++++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/search.php b/search.php index 25497d6..8fa24d7 100644 --- a/search.php +++ b/search.php @@ -69,7 +69,6 @@ } ?>
-
Date: Mon, 28 Aug 2023 18:46:29 -0400 Subject: [PATCH 35/40] add instances, fix formatting --- instances.json | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/instances.json b/instances.json index 38e8029..80dbc55 100644 --- a/instances.json +++ b/instances.json @@ -2,22 +2,22 @@ "instances": [ { "clearnet": "https://search.ahwx.org/", - "tor": "http://hyy7rcvknwb22v4nnoar635wntiwr4uwzhiuyimemyl4fz6k7tahj5id.onion", + "tor": "http://hyy7rcvknwb22v4nnoar635wntiwr4uwzhiuyimemyl4fz6k7tahj5id.onion/", "i2p": null, "country": "NL", "librey": true }, { "clearnet": "https://librex.me/", - "tor": "http://librex.revvybrr6pvbx4n3j4475h4ghw4elqr4t5xo2vtd3gfpu2nrsnhh57id.onion", - "i2p": "http://revekebotog64xrrammtsmjwtwlg3vqyzwdurzt2pu6botg4bejq.b32.i2p", + "tor": "http://librex.revvybrr6pvbx4n3j4475h4ghw4elqr4t5xo2vtd3gfpu2nrsnhh57id.onion/", + "i2p": "http://revekebotog64xrrammtsmjwtwlg3vqyzwdurzt2pu6botg4bejq.b32.i2p/", "country": "CA", "librey": true }, { "clearnet": "https://librex.revvy.de/", - "tor": "http://librex.revvybrr6pvbx4n3j4475h4ghw4elqr4t5xo2vtd3gfpu2nrsnhh57id.onion", - "i2p": "http://revekebotog64xrrammtsmjwtwlg3vqyzwdurzt2pu6botg4bejq.b32.i2p", + "tor": "http://librex.revvybrr6pvbx4n3j4475h4ghw4elqr4t5xo2vtd3gfpu2nrsnhh57id.onion/", + "i2p": "http://revekebotog64xrrammtsmjwtwlg3vqyzwdurzt2pu6botg4bejq.b32.i2p/", "country": "CA", "librey": true }, @@ -28,6 +28,13 @@ "country": "CL", "librey": true }, + { + "clearnet": "https://librey.org/", + "tor": "http://jxhkfulu6wpdl4apuy4dyivuowmpprvsd7e3el2z73crq7fmyv7rjkyd.onion/", + "i2p": null, + "country": "US", + "librey": true + }, { "clearnet": "https://search.davidovski.xyz/", "tor": null, From da95ec8c9e90fc406fb31d7c6277e551df5d811a Mon Sep 17 00:00:00 2001 From: nohoster <136514837+nohoster@users.noreply.github.com> Date: Tue, 29 Aug 2023 01:51:35 +0000 Subject: [PATCH 36/40] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 44e781a..312a5db 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ You can access the full list of LibreX and LibreY instances on one of the follow | [librex.me](https://librex.me/) | [✅](http://librex.revvybrr6pvbx4n3j4475h4ghw4elqr4t5xo2vtd3gfpu2nrsnhh57id.onion/) | [✅](http://revekebotog64xrrammtsmjwtwlg3vqyzwdurzt2pu6botg4bejq.b32.i2p/) | 🇨🇦 CA | | [librex.revvy.de](https://librex.revvy.de/) | [✅](http://librex.revvybrr6pvbx4n3j4475h4ghw4elqr4t5xo2vtd3gfpu2nrsnhh57id.onion/) | [✅](http://revekebotog64xrrammtsmjwtwlg3vqyzwdurzt2pu6botg4bejq.b32.i2p/) | 🇨🇦 CA | | [search.davidovski.xyz](https://search.davidovski.xyz/) | ❌ | ❌ | 🇬🇧 GB | +| [librey.nohost.network](https://librey.nohost.network/) | ❌ | ❌ | 🇲🇽 MX |
From 08bb5b5d49e293013417856dc14e36147a00ff07 Mon Sep 17 00:00:00 2001 From: nohoster <136514837+nohoster@users.noreply.github.com> Date: Tue, 29 Aug 2023 01:53:33 +0000 Subject: [PATCH 37/40] Update instances.json --- instances.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/instances.json b/instances.json index 80dbc55..5889c4a 100644 --- a/instances.json +++ b/instances.json @@ -70,6 +70,13 @@ "country": "DE", "librey": true }, + { + "clearnet": "https://librex.nohost.network/", + "tor": null, + "i2p": null, + "country": "MX", + "librey": true + }, { "clearnet": "https://lx.vern.cc/", "tor": "http://lx.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion/", @@ -140,13 +147,6 @@ "country": "DE", "librey": false }, - { - "clearnet": "https://librex.nohost.network/", - "tor": null, - "i2p": null, - "country": "MX", - "librey": false - }, { "clearnet": "https://librex.pardesicat.xyz/", "tor": null, From 13304533704b385a0089a52c29a54759299fd847 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Dobo=C5=A1?= <26146502+kubo6472@users.noreply.github.com> Date: Tue, 29 Aug 2023 10:09:18 +0200 Subject: [PATCH 38/40] fix unable to create organization package --- .github/workflows/docker-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 1427a4d..62675ca 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -6,6 +6,7 @@ on: jobs: build: runs-on: ubuntu-latest + permissions: write-all steps: - uses: actions/checkout@v3 - uses: docker/login-action@v2.1.0 From a2f94fb0caff99a968f9e1471bb558b7769306e5 Mon Sep 17 00:00:00 2001 From: nohoster <136514837+nohoster@users.noreply.github.com> Date: Tue, 29 Aug 2023 14:04:15 +0000 Subject: [PATCH 39/40] Fix variable format. The format was wrong, it had a misplaced "}". It should be fine now. --- docker/attributes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/attributes.sh b/docker/attributes.sh index 95a1c8c..8c4c8b1 100755 --- a/docker/attributes.sh +++ b/docker/attributes.sh @@ -23,7 +23,7 @@ export CONFIG_GOOGLE_DOMAIN="${CONFIG_GOOGLE_DOMAIN:-"com"}" export CONFIG_GOOGLE_LANGUAGE_SITE="${CONFIG_GOOGLE_LANGUAGE_SITE:-"en"}" export CONFIG_GOOGLE_LANGUAGE_RESULTS="${CONFIG_GOOGLE_LANGUAGE_RESULTS:-"en"}" export CONFIG_GOOGLE_NUMBER_OF_RESULTS="${CONFIG_GOOGLE_NUMBER_OF_RESULTS:-"10"}" -export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK}:-true}" +export CONFIG_INSTANCE_FALLBACK="${CONFIG_INSTANCE_FALLBACK:-true}" export CONFIG_INVIDIOUS_INSTANCE="${CONFIG_INVIDIOUS_INSTANCE:-"invidious.snopyta.org"}" export CONFIG_HIDDEN_SERVICE_SEARCH=${CONFIG_HIDDEN_SERVICE_SEARCH:-false} export CONFIG_DISABLE_BITTORRENT_SEARCH=${CONFIG_DISABLE_BITTORRENT_SEARCH:-false} From 33240c4d21efbe15bb15fd9d742fae87bf9c8a87 Mon Sep 17 00:00:00 2001 From: Revvy Date: Tue, 29 Aug 2023 14:07:50 -0400 Subject: [PATCH 40/40] fix high-severity vulnerability --- image_proxy.php | 2 +- misc/tools.php | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/image_proxy.php b/image_proxy.php index bfeab02..9b5f775 100644 --- a/image_proxy.php +++ b/image_proxy.php @@ -6,7 +6,7 @@ $url = $_REQUEST["url"]; $requested_root_domain = get_root_domain($url); - $allowed_domains = array("qwant.com", "wikimedia.org", get_root_domain($config->invidious_instance_for_video_results)); + $allowed_domains = array("s2.qwant.com", "s1.qwant.com", "upload.wikimedia.org", get_root_domain($config->invidious_instance_for_video_results)); if (in_array($requested_root_domain, $allowed_domains)) { diff --git a/misc/tools.php b/misc/tools.php index 20ffa9b..f210bf0 100644 --- a/misc/tools.php +++ b/misc/tools.php @@ -6,13 +6,7 @@ } function get_root_domain($url) { - $split_url = explode("/", $url); - $base_url = $split_url[2]; - - $base_url_main_split = explode(".", strrev($base_url)); - $root_domain = strrev($base_url_main_split[1]) . "." . strrev($base_url_main_split[0]); - - return $root_domain; + return parse_url($url, PHP_URL_HOST); } function try_replace_with_frontend($url, $frontend, $original, $opts) {