search.suspended_time settings: bug fixes

Browse Source 
* fix type in settings.yml: replace suspend_times by suspended_times
* always use delay defined in settings.yml:
  * HTTP status 402 and 403: read the value from settings.yml instead of using the hardcoded value of 1 day.
  * startpage engine: CAPTCHA suspend the engine for one day instead of one week
This commit is contained in:
Alexandre Flament 2023-01-28 10:24:14 +00:00
parent 6b71721ce8
commit 37addec69e
4 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
searx/engines/startpage.py
View File

@ -62,8 +62,7 @@ sc_code = ''
def raise_captcha(resp): def raise_captcha(resp):
if str(resp.url).startswith('https://www.startpage.com/sp/captcha'): if str(resp.url).startswith('https://www.startpage.com/sp/captcha'):
# suspend CAPTCHA for 7 days raise SearxEngineCaptchaException()
raise SearxEngineCaptchaException(suspended_time=7 * 24 * 3600)
def get_sc_code(headers): def get_sc_code(headers):

9
searx/exceptions.py
View File

@ -70,8 +70,15 @@ class SearxEngineAccessDeniedException(SearxEngineResponseException):
"""The website is blocking the access""" """The website is blocking the access"""
SUSPEND_TIME_SETTING = "search.suspended_times.SearxEngineAccessDenied" SUSPEND_TIME_SETTING = "search.suspended_times.SearxEngineAccessDenied"
"""This settings contains the default suspended time"""
def __init__(self, suspended_time=None, message='Access denied'): def __init__(self, suspended_time: int = None, message: str = 'Access denied'):
"""Generic exception to raise when an engine denies access to the results
Args:
suspended_time (int, optional): How long the engine is going to be suspended in second. Defaults to None.
message (str, optional): Internal message. Defaults to 'Access denied'.
"""
suspended_time = suspended_time or self._get_default_suspended_time() suspended_time = suspended_time or self._get_default_suspended_time()
super().__init__(message + ', suspended_time=' + str(suspended_time)) super().__init__(message + ', suspended_time=' + str(suspended_time))
self.suspended_time = suspended_time self.suspended_time = suspended_time

4
searx/network/raise_for_httperror.py
View File

@ -72,9 +72,7 @@ def raise_for_httperror(resp):
if resp.status_code and resp.status_code >= 400: if resp.status_code and resp.status_code >= 400:
raise_for_captcha(resp) raise_for_captcha(resp)
if resp.status_code in (402, 403): if resp.status_code in (402, 403):
raise SearxEngineAccessDeniedException( raise SearxEngineAccessDeniedException(message='HTTP error ' + str(resp.status_code))
message='HTTP error ' + str(resp.status_code), suspended_time=3600 * 24
)
if resp.status_code == 429: if resp.status_code == 429:
raise SearxEngineTooManyRequestsException() raise SearxEngineTooManyRequestsException()
resp.raise_for_status() resp.raise_for_status()

2
searx/settings.yml
View File

@ -45,7 +45,7 @@ search:
ban_time_on_fail: 5 ban_time_on_fail: 5
# max ban time in seconds after engine errors # max ban time in seconds after engine errors
max_ban_time_on_fail: 120 max_ban_time_on_fail: 120
suspend_times: suspended_times:
# Engine suspension time after error (in seconds; set to 0 to disable) # Engine suspension time after error (in seconds; set to 0 to disable)
# For error "Access denied" and "HTTP error [402, 403]" # For error "Access denied" and "HTTP error [402, 403]"
SearxEngineAccessDenied: 86400 SearxEngineAccessDenied: 86400