[fix] Gigablast.com has been erased

Browse Source 
[1] https://www.reddit.com/r/searchengines/comments/128wdcp/gigablastcom_has_been_erased/

Closes: https://github.com/searxng/searxng/issues/2321
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
Markus Heiser 2023-04-06 08:22:57 +02:00
parent 934249dd05
commit 5234e45010
2 changed files with 0 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
searx/engines/gigablast.py
View File

@ -1,133 +0,0 @@
# SPDX-License-Identifier: AGPL-3.0-or-later
# lint: pylint
"""
Gigablast (Web)
"""
# pylint: disable=invalid-name, use-dict-literal
import re
from time import time
from json import loads
from urllib.parse import urlencode
from searx.network import get
# about
about = {
"website": 'https://www.gigablast.com',
"wikidata_id": 'Q3105449',
"official_api_documentation": 'https://gigablast.com/api.html',
"use_official_api": True,
"require_api_key": False,
"results": 'JSON',
}
# engine dependent config
categories = ['general', 'web']
# gigablast's pagination is totally damaged, don't use it
paging = False
safesearch = True
# search-url
base_url = 'https://gigablast.com'
search_path = '/search?'
# ugly hack: gigablast requires a random extra parameter which can be extracted
# from the source code of the gigablast HTTP client
extra_param = ''
# timestamp of the last fetch of extra_param
extra_param_ts = 0
# after how many seconds extra_param expire
extra_param_expiration_delay = 3000
gb_userid = ''
gb_code = ''
def fetch_extra_param(query_args, headers):
# example:
#
# var uxrl='/search?c=main&qlangcountry=en-us&q=south&s=10&rand=1590740241635&n';
# uxrl=uxrl+'sab=730863287';
#
# extra_param --> "rand=1590740241635&nsab=730863287"
global extra_param, extra_param_ts # pylint: disable=global-statement
extra_param_ts = time()
extra_param_path = search_path + urlencode(query_args)
text = get(base_url + extra_param_path, headers=headers).text
re_var = None
for line in text.splitlines():
if re_var is None and extra_param_path in line:
var = line.split("=")[0].split()[1] # e.g. var --> 'uxrl'
re_var = re.compile(var + "\\s*=\\s*" + var + "\\s*\\+\\s*'" + "(.*)" + "'(.*)")
extra_param = line.split("'")[1][len(extra_param_path) :]
continue
if re_var is not None and re_var.search(line):
extra_param += re_var.search(line).group(1)
break
# do search-request
def request(query, params): # pylint: disable=unused-argument
query_args = dict(c='main', q=query, dr=1, showgoodimages=0)
if gb_userid and gb_code:
query_args['userid'] = gb_userid
query_args['code'] = gb_code
if params['language'] and params['language'] != 'all':
query_args['qlangcountry'] = params['language']
query_args['qlang'] = params['language'].split('-')[0]
if params['safesearch'] >= 1:
query_args['ff'] = 1
# see API http://www.gigablast.com/api.html#/search
# Take into account, that the API has some quirks ..
if time() > (extra_param_ts + extra_param_expiration_delay):
fetch_extra_param(query_args, params['headers'])
query_args['format'] = 'json'
params['url'] = base_url + search_path + urlencode(query_args) + extra_param
return params
# get response from search-request
def response(resp):
results = []
response_json = loads(resp.text)
# logger.debug('gigablast returns %s results', len(response_json['results']))
for result in response_json['results']:
# see "Example JSON Output (&format=json)"
# at http://www.gigablast.com/api.html#/search
# sort out meaningless result
title = result.get('title')
if len(title) < 2:
continue
url = result.get('url')
if len(url) < 9:
continue
content = result.get('sum')
if len(content) < 5:
continue
# extend fields
subtitle = result.get('title')
if len(subtitle) > 3 and subtitle != title:
title += " - " + subtitle
results.append(dict(url=url, title=title, content=content))
return results

11
searx/settings.yml
View File

@ -670,17 +670,6 @@ engines:
engine: genius
shortcut: gen
- name: gigablast
engine: gigablast
shortcut: gb
timeout: 4.0
# API key required, see https://gigablast.com/searchfeed.html
# gb_userid: unset
# gb_code: unknown
disabled: true
additional_tests:
rosebud: *test_rosebud
- name: gentoo
engine: gentoo
shortcut: ge