Fijxu/librex
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fallback on all queries

Browse source
This commit is contained in:
davidovski 2023-08-23 00:38:59 +01:00
parent 649b0065e3
commit 1b898c6614
14 changed files with 99 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
api.php
View file

@ -1,10 +1,10 @@
<?php
$config = require "config.php";
require "misc/tools.php";
require "misc/search_engine.php";
if (!isset($_REQUEST["q"]))
{
$opts = load_opts();
if (!$opts->query) {
echo "<p>Example API request: <a href=\"./api.php?q=gentoo&p=2&t=0\">./api.php?q=gentoo&p=2&t=0</a></p>
<br/>
<p>\"q\" is the keyword</p>
@ -17,12 +17,7 @@
die();
}
$query = $_REQUEST["q"];
$query_encoded = urlencode($query);
$page = isset($_REQUEST["p"]) ? (int) $_REQUEST["p"] : 0;
$type = isset($_REQUEST["t"]) ? (int) $_REQUEST["t"] : 0;
$results = fetch_search_results($type, $query, $page, $config, false);
$results = fetch_search_results($opts, false);
header("Content-Type: application/json");
echo json_encode($results);
?>

4
engines/ahmia/hidden_service.php
View file

@ -8,9 +8,11 @@
public function get_results() {
$response = curl_multi_getcontent($this->ch);
$results = array();
$xpath = get_xpath($response);
$results = array();
if (!$xpath)
return $results;
foreach($xpath->query("//ol[@class='searchResults']//li[@class='result']") as $result)
{

3
engines/bittorrent/1337x.php
View file

@ -11,6 +11,9 @@
$xpath = get_xpath($response);
$results = array();
if (!$xpath)
return $results;
foreach($xpath->query("//table/tbody/tr") as $result) {
$name = $xpath->evaluate(".//td[@class='coll-1 name']/a", $result)[1]->textContent;
$magnet = "./engines/bittorrent/get_magnet_1337x.php?url=https://1337x.to" . $xpath->evaluate(".//td[@class='coll-1 name']/a/@href", $result)[1]->textContent;

3
engines/bittorrent/nyaa.php
View file

@ -11,6 +11,9 @@
$xpath = get_xpath($response);
$results = array();
if (!$xpath)
return $results;
foreach($xpath->query("//tbody/tr") as $result)
{
$name_node = $xpath->evaluate(".//td[@colspan='2']//a[not(contains(@class, 'comments'))]/@title", $result);

2
engines/bittorrent/rutor.php
View file

@ -9,6 +9,8 @@
$xpath = get_xpath($response);
$results = array();
if (!$xpath)
return $results;
foreach($xpath->query("//table/tr[@class='gai' or @class='tum']") as $result)
{

53
engines/bittorrent/torrentgalaxy.php
View file

@ -6,33 +6,36 @@
}
public function get_results() {
$response = curl_multi_getcontent($this->ch);
$xpath = get_xpath($response);
$results = array();
foreach($xpath->query("//div[@class='tgxtablerow txlight']") as $result)
{
$name = $xpath->evaluate(".//div[contains(@class, 'clickable-row')]", $result)[0]->textContent;
$magnet = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/a/@href", $result)[1]->textContent;
$magnet_without_tracker = explode("&tr=", $magnet)[0];
$magnet = $magnet_without_tracker . $this->opts->bittorrent_trackers;
$size = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span", $result)[0]->textContent;
$seeders = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[1]->textContent;
$leechers = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[2]->textContent;
array_push($results,
array (
"name" => htmlspecialchars($name),
"seeders" => (int) $seeders,
"leechers" => (int) $leechers,
"magnet" => htmlspecialchars($magnet),
"size" => htmlspecialchars($size),
"source" => "torrentgalaxy.to"
)
);
}
$response = curl_multi_getcontent($this->ch);
$xpath = get_xpath($response);
$results = array();
if (!$xpath)
return $results;
foreach($xpath->query("//div[@class='tgxtablerow txlight']") as $result)
{
$name = $xpath->evaluate(".//div[contains(@class, 'clickable-row')]", $result)[0]->textContent;
$magnet = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/a/@href", $result)[1]->textContent;
$magnet_without_tracker = explode("&tr=", $magnet)[0];
$magnet = $magnet_without_tracker . $this->opts->bittorrent_trackers;
$size = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span", $result)[0]->textContent;
$seeders = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[1]->textContent;
$leechers = $xpath->evaluate(".//div[@class='tgxtablecell collapsehide rounded txlight']/span/font", $result)[2]->textContent;
array_push($results,
array (
"name" => htmlspecialchars($name),
"seeders" => (int) $seeders,
"leechers" => (int) $leechers,
"magnet" => htmlspecialchars($magnet),
"size" => htmlspecialchars($size),
"source" => "torrentgalaxy.to"
)
);
}
return $results;
}
}
?>

2
engines/invidious/video.php
View file

@ -1,7 +1,7 @@
<?php
class VideoSearch extends EngineRequest {
public function get_request_url() {
$this->instance_url = $this->config->invidious_instance_for_video_results;
$this->instance_url = $this->opts->invidious_instance_for_video_results;
$query = urlencode($this->query);
return "$this->instance_url/api/v1/search?q=$query";
}

55
engines/librex/fallback.php
View file

@ -1,25 +1,28 @@
<?php
class LibreXFallback extends EngineRequest {
public function __construct($instance, $opts, $mh) {
$this->instance = $instance;
parent::__construct($opts, $mh);
class LibreXFallback extends EngineRequest {
public function __construct($instance, $opts, $mh) {
$this->instance = $instance;
parent::__construct($opts, $mh);
}
public function get_request_url() {
return $this->instance . "api.php?" . opts_to_params($this->opts);
}
public function get_results() {
$response = curl_exec($this->ch);
$response = json_decode($response, true);
if (!$response)
return array();
return array_values($response);
}
}
public function get_request_url() {
return $instance . "api.php?" .opts_to_params($opts);
}
}
function get_librex_results($opts, $mh) {
global $config;
if (isset($_REQUEST["nfb"]) && $_REQUEST["nfb"] == "1")
return array();
if (!$config->instance_fallback)
function get_librex_results($opts) {
if (!$opts->do_fallback)
return array();
$instances_json = json_decode(file_get_contents("instances.json"), true);
@ -27,12 +30,11 @@ class LibreXFallback extends EngineRequest {
if (empty($instances_json["instances"]))
return array();
// TODO pick instances which aren't on cooldown
$instances = array_map(fn($n) => $n['clearnet'], array_filter($instances_json['instances'], fn($n) => !is_null($n['clearnet'])));
shuffle($instances);
$query_encoded = urlencode($query);
$results = array();
$tries = 0;
@ -44,18 +46,13 @@ class LibreXFallback extends EngineRequest {
if (parse_url($instance)["host"] == parse_url($_SERVER['HTTP_HOST'])["host"])
continue;
$url = $instance . "api.php?q=$query_encoded&p=$page&t=0&nfb=1";
$librex_request = new LibreXFallback($instance, $opts, null);
$results = $librex_request->get_results();
$librex_ch = curl_init($url);
curl_setopt_array($librex_ch, $config->curl_settings);
copy_cookies($librex_ch);
$response = curl_exec($librex_ch);
curl_close($librex_ch);
if (count($results) > 1)
return $results;
$code = curl_getinfo($librex_ch)["http_code"];
$results = json_decode($response, true);
} while ( !empty($instances) && ($results == null || count($results) <= 1));
} while ( !empty($instances));
if (empty($instances))
return array();

3
engines/qwant/image.php
View file

@ -11,6 +11,9 @@
$results = array();
$xpath = get_xpath(curl_multi_getcontent($this->ch));
if (!$xpath)
return $results;
foreach($xpath->query("//a[@rel='noopener']") as $result)
{
$image = $xpath->evaluate(".//img", $result)[0];

2
engines/text/duckduckgo.php
View file

@ -57,7 +57,7 @@
)
);
}
return $results;
return $results;
}
}

2
engines/text/google.php
View file

@ -51,7 +51,7 @@
}
$url = $url->textContent;
$url = check_for_privacy_frontend($url, $opts);
$url = check_for_privacy_frontend($url, $this->opts);
$title = $xpath->evaluate(".//h3", $result)[0];
$description = $xpath->evaluate(".//div[contains(@class, 'VwiC3b')]", $result)[0];

7
engines/text/text.php
View file

@ -27,11 +27,8 @@
}
public function get_results() {
if (curl_getinfo($this->engine_request->ch)['http_code'] != '200')
{
require "engines/librex/text.php";
return get_librex_results($this->query, $this->page);
}
if (curl_getinfo($this->engine_request->ch)['http_code'] != '200')
return array();
$results = $this->engine_request->get_results();

24
misc/search_engine.php
View file

@ -6,17 +6,21 @@
$this->opts = $opts;
$url = $this->get_request_url();
error_log($url);
if ($url) {
$this->ch = curl_init($url);
curl_setopt_array($this->ch, $opts->curl_settings);
curl_multi_add_handle($mh, $this->ch);
if ($opts->curl_settings)
curl_setopt_array($this->ch, $opts->curl_settings);
if ($mh)
curl_multi_add_handle($mh, $this->ch);
}
}
public function get_request_url(){
return "";
}
public function successful() {
return curl_getinfo($this->ch)['http_code'] == '200';
}
@ -41,7 +45,7 @@
$opts->disable_frontends = (int) ($_REQUEST["nf"] ?? 0) == 1 || isset($_COOKIE["disable_frontends"]);
$opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"]));
$opts->language = $_REQUEST["lang"] ?? trim(htmlspecialchars($_COOKIE["language"] ?? ""));
$opts->do_fallback = (int) ($_REQUEST["nfb"] ?? 0) == 0;
if (!$opts->instance_fallback) {
@ -63,10 +67,10 @@
$params .= "p=$opts->page";
$params .= "&q=$query";
$params .= "&t=$opts->type";
$params .= "&nfb=" . $opts->do_fallback ? 0 : 1;
$params .= "&safe=" . $opts->safe_search ? 1 : 0;
$params .= "&nf=" . $opts->disable_frontends ? 1 : 0;
$params .= "&ns=" . $opts->disable_special ? 1 : 0;
$params .= "&nfb=" . ($opts->do_fallback ? 0 : 1);
$params .= "&safe=" . ($opts->safe_search ? 1 : 0);
$params .= "&nf=" . ($opts->disable_frontends ? 1 : 0);
$params .= "&ns=" . ($opts->disable_special ? 1 : 0);
return $params;
}
@ -119,8 +123,8 @@
$results = $search_category->get_results();
if (empty($results)) {
//if (!$opt->do_fallback)
//return ;
require "engines/librex/fallback.php";
$results = get_librex_results($opts);
}
if (!$do_print)

15
misc/tools.php
View file

@ -17,14 +17,12 @@
return $root_domain;
}
function try_replace_with_frontend($url, $frontend, $original)
{
global $config;
function try_replace_with_frontend($url, $frontend, $original, $opts) {
$frontends = $opts->frontends;
$frontend = $opts->$frontend["instance_url"];
$frontend = $opts->$frontend;
if ($frontend) {
$frontend = $frontend->instance_url;
if (empty(trim($frontend)))
return $url;
@ -67,10 +65,10 @@
$original = $data["original_url"];
if (strpos($url, $original)) {
$url = try_replace_with_frontend($url, $frontend, $original);
$url = try_replace_with_frontend($url, $frontend, $original, $opts);
break;
} else if (strpos($url, "stackexchange.com")) {
$url = try_replace_with_frontend($url, "anonymousoverflow", "stackexchange.com");
$url = try_replace_with_frontend($url, "anonymousoverflow", "stackexchange.com", $opts);
break;
}
}
@ -79,6 +77,9 @@
}
function get_xpath($response) {
if (!$response)
return null;
$htmlDom = new DOMDocument;
@$htmlDom->loadHTML($response);
$xpath = new DOMXPath($htmlDom);