[fix] Gigablast.com has been erased
[1] https://www.reddit.com/r/searchengines/comments/128wdcp/gigablastcom_has_been_erased/ Closes: https://github.com/searxng/searxng/issues/2321 Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
This commit is contained in:
		
							parent
							
								
									934249dd05
								
							
						
					
					
						commit
						5234e45010
					
				| @ -1,133 +0,0 @@ | |||||||
| # SPDX-License-Identifier: AGPL-3.0-or-later |  | ||||||
| # lint: pylint |  | ||||||
| """ |  | ||||||
|  Gigablast (Web) |  | ||||||
| """ |  | ||||||
| # pylint: disable=invalid-name, use-dict-literal |  | ||||||
| 
 |  | ||||||
| import re |  | ||||||
| from time import time |  | ||||||
| from json import loads |  | ||||||
| from urllib.parse import urlencode |  | ||||||
| from searx.network import get |  | ||||||
| 
 |  | ||||||
| # about |  | ||||||
| about = { |  | ||||||
|     "website": 'https://www.gigablast.com', |  | ||||||
|     "wikidata_id": 'Q3105449', |  | ||||||
|     "official_api_documentation": 'https://gigablast.com/api.html', |  | ||||||
|     "use_official_api": True, |  | ||||||
|     "require_api_key": False, |  | ||||||
|     "results": 'JSON', |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| # engine dependent config |  | ||||||
| categories = ['general', 'web'] |  | ||||||
| # gigablast's pagination is totally damaged, don't use it |  | ||||||
| paging = False |  | ||||||
| safesearch = True |  | ||||||
| 
 |  | ||||||
| # search-url |  | ||||||
| base_url = 'https://gigablast.com' |  | ||||||
| search_path = '/search?' |  | ||||||
| 
 |  | ||||||
| # ugly hack: gigablast requires a random extra parameter which can be extracted |  | ||||||
| # from the source code of the gigablast HTTP client |  | ||||||
| extra_param = '' |  | ||||||
| # timestamp of the last fetch of extra_param |  | ||||||
| extra_param_ts = 0 |  | ||||||
| # after how many seconds extra_param expire |  | ||||||
| extra_param_expiration_delay = 3000 |  | ||||||
| 
 |  | ||||||
| gb_userid = '' |  | ||||||
| gb_code = '' |  | ||||||
| 
 |  | ||||||
| 
 |  | ||||||
| def fetch_extra_param(query_args, headers): |  | ||||||
| 
 |  | ||||||
|     # example: |  | ||||||
|     # |  | ||||||
|     # var uxrl='/search?c=main&qlangcountry=en-us&q=south&s=10&rand=1590740241635&n'; |  | ||||||
|     # uxrl=uxrl+'sab=730863287'; |  | ||||||
|     # |  | ||||||
|     # extra_param --> "rand=1590740241635&nsab=730863287" |  | ||||||
| 
 |  | ||||||
|     global extra_param, extra_param_ts  # pylint: disable=global-statement |  | ||||||
| 
 |  | ||||||
|     extra_param_ts = time() |  | ||||||
|     extra_param_path = search_path + urlencode(query_args) |  | ||||||
|     text = get(base_url + extra_param_path, headers=headers).text |  | ||||||
| 
 |  | ||||||
|     re_var = None |  | ||||||
|     for line in text.splitlines(): |  | ||||||
|         if re_var is None and extra_param_path in line: |  | ||||||
|             var = line.split("=")[0].split()[1]  # e.g. var --> 'uxrl' |  | ||||||
|             re_var = re.compile(var + "\\s*=\\s*" + var + "\\s*\\+\\s*'" + "(.*)" + "'(.*)") |  | ||||||
|             extra_param = line.split("'")[1][len(extra_param_path) :] |  | ||||||
|             continue |  | ||||||
|         if re_var is not None and re_var.search(line): |  | ||||||
|             extra_param += re_var.search(line).group(1) |  | ||||||
|             break |  | ||||||
| 
 |  | ||||||
| 
 |  | ||||||
| # do search-request |  | ||||||
| def request(query, params):  # pylint: disable=unused-argument |  | ||||||
|     query_args = dict(c='main', q=query, dr=1, showgoodimages=0) |  | ||||||
| 
 |  | ||||||
|     if gb_userid and gb_code: |  | ||||||
|         query_args['userid'] = gb_userid |  | ||||||
|         query_args['code'] = gb_code |  | ||||||
| 
 |  | ||||||
|     if params['language'] and params['language'] != 'all': |  | ||||||
|         query_args['qlangcountry'] = params['language'] |  | ||||||
|         query_args['qlang'] = params['language'].split('-')[0] |  | ||||||
| 
 |  | ||||||
|     if params['safesearch'] >= 1: |  | ||||||
|         query_args['ff'] = 1 |  | ||||||
| 
 |  | ||||||
|     # see API http://www.gigablast.com/api.html#/search |  | ||||||
|     # Take into account, that the API has some quirks .. |  | ||||||
|     if time() > (extra_param_ts + extra_param_expiration_delay): |  | ||||||
|         fetch_extra_param(query_args, params['headers']) |  | ||||||
| 
 |  | ||||||
|     query_args['format'] = 'json' |  | ||||||
|     params['url'] = base_url + search_path + urlencode(query_args) + extra_param |  | ||||||
| 
 |  | ||||||
|     return params |  | ||||||
| 
 |  | ||||||
| 
 |  | ||||||
| # get response from search-request |  | ||||||
| def response(resp): |  | ||||||
|     results = [] |  | ||||||
| 
 |  | ||||||
|     response_json = loads(resp.text) |  | ||||||
| 
 |  | ||||||
|     # logger.debug('gigablast returns %s results', len(response_json['results'])) |  | ||||||
| 
 |  | ||||||
|     for result in response_json['results']: |  | ||||||
|         # see "Example JSON Output (&format=json)" |  | ||||||
|         # at http://www.gigablast.com/api.html#/search |  | ||||||
| 
 |  | ||||||
|         # sort out meaningless result |  | ||||||
| 
 |  | ||||||
|         title = result.get('title') |  | ||||||
|         if len(title) < 2: |  | ||||||
|             continue |  | ||||||
| 
 |  | ||||||
|         url = result.get('url') |  | ||||||
|         if len(url) < 9: |  | ||||||
|             continue |  | ||||||
| 
 |  | ||||||
|         content = result.get('sum') |  | ||||||
|         if len(content) < 5: |  | ||||||
|             continue |  | ||||||
| 
 |  | ||||||
|         # extend fields |  | ||||||
| 
 |  | ||||||
|         subtitle = result.get('title') |  | ||||||
|         if len(subtitle) > 3 and subtitle != title: |  | ||||||
|             title += " - " + subtitle |  | ||||||
| 
 |  | ||||||
|         results.append(dict(url=url, title=title, content=content)) |  | ||||||
| 
 |  | ||||||
|     return results |  | ||||||
| @ -670,17 +670,6 @@ engines: | |||||||
|     engine: genius |     engine: genius | ||||||
|     shortcut: gen |     shortcut: gen | ||||||
| 
 | 
 | ||||||
|   - name: gigablast |  | ||||||
|     engine: gigablast |  | ||||||
|     shortcut: gb |  | ||||||
|     timeout: 4.0 |  | ||||||
|     # API key required, see https://gigablast.com/searchfeed.html |  | ||||||
|     # gb_userid: unset |  | ||||||
|     # gb_code: unknown |  | ||||||
|     disabled: true |  | ||||||
|     additional_tests: |  | ||||||
|       rosebud: *test_rosebud |  | ||||||
| 
 |  | ||||||
|   - name: gentoo |   - name: gentoo | ||||||
|     engine: gentoo |     engine: gentoo | ||||||
|     shortcut: ge |     shortcut: ge | ||||||
|  | |||||||
		Loading…
	
		Reference in New Issue
	
	Block a user