[mod] checker : replace pycld3 by langdetect

Browse Source 
pycld3 requires the native library cld3
langdetect is a pure python package
This commit is contained in:
Alexandre Flament 2021-01-19 21:26:04 +01:00
parent 0495e15df4
commit aa887eb375
3 changed files with 13 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@ -9,4 +9,4 @@ pygments==2.1.3
python-dateutil==2.8.1 python-dateutil==2.8.1
pyyaml==5.3.1 pyyaml==5.3.1
requests[socks]==2.25.1 requests[socks]==2.25.1
pycld3==0.20 langdetect==1.0.8

15
searx/search/checker/impl.py
View File

@ -9,7 +9,8 @@ from time import time
from urllib.parse import urlparse from urllib.parse import urlparse
import re import re
import cld3 from langdetect import detect_langs
from langdetect.lang_detect_exception import LangDetectException
import requests.exceptions import requests.exceptions
from searx import poolrequests, logger from searx import poolrequests, logger
@ -181,10 +182,14 @@ class ResultContainerTests:
self.test_results.add_error(self.test_name, message, *args, '(' + sqstr + ')') self.test_results.add_error(self.test_name, message, *args, '(' + sqstr + ')')
def _add_language(self, text: str) -> typing.Optional[str]: def _add_language(self, text: str) -> typing.Optional[str]:
r = cld3.get_language(str(text)) # pylint: disable=E1101 try:
if r is not None and r.probability >= 0.98 and r.is_reliable: r = detect_langs(str(text)) # pylint: disable=E1101
self.languages.add(r.language) except LangDetectException:
self.test_results.add_language(r.language) return None
if len(r) > 0 and r[0].prob > 0.95:
self.languages.add(r[0].lang)
self.test_results.add_language(r[0].lang)
return None return None
def _check_result(self, result): def _check_result(self, result):

6
utils/searx.sh
View File

@ -46,7 +46,6 @@ SEARX_PACKAGES_debian="\
python3-dev python3-babel python3-venv python3-dev python3-babel python3-venv
uwsgi uwsgi-plugin-python3 uwsgi uwsgi-plugin-python3
git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
libprotobuf-dev protobuf-compiler
shellcheck" shellcheck"
BUILD_PACKAGES_debian="\ BUILD_PACKAGES_debian="\
@ -59,7 +58,6 @@ SEARX_PACKAGES_arch="\
python python-pip python-lxml python-babel python python-pip python-lxml python-babel
uwsgi uwsgi-plugin-python uwsgi uwsgi-plugin-python
git base-devel libxml2 git base-devel libxml2
protobuf
shellcheck" shellcheck"
BUILD_PACKAGES_arch="\ BUILD_PACKAGES_arch="\
@ -71,7 +69,7 @@ SEARX_PACKAGES_fedora="\
python python-pip python-lxml python-babel python python-pip python-lxml python-babel
uwsgi uwsgi-plugin-python3 uwsgi uwsgi-plugin-python3
git @development-tools libxml2 git @development-tools libxml2
ShellCheck protobuf-compiler protobuf-devel" ShellCheck"
BUILD_PACKAGES_fedora="\ BUILD_PACKAGES_fedora="\
firefox graphviz graphviz-gd ImageMagick librsvg2-tools firefox graphviz graphviz-gd ImageMagick librsvg2-tools
@ -84,7 +82,7 @@ SEARX_PACKAGES_centos="\
python36 python36-pip python36-lxml python-babel python36 python36-pip python36-lxml python-babel
uwsgi uwsgi-plugin-python3 uwsgi uwsgi-plugin-python3
git @development-tools libxml2 git @development-tools libxml2
ShellCheck protobuf-compiler protobuf-devel" ShellCheck"
BUILD_PACKAGES_centos="\ BUILD_PACKAGES_centos="\
firefox graphviz graphviz-gd ImageMagick librsvg2-tools firefox graphviz graphviz-gd ImageMagick librsvg2-tools