[refactor] translation engines: common interface

Browse Source
This commit is contained in:
Bnyro 2024-10-15 15:39:14 +02:00 committed by Markus Heiser
parent 3742d558ac
commit 34b8def91c
8 changed files with 125 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
searx/engines/deepl.py
View File

@ -1,8 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
"""Deepl translation engine""" """Deepl translation engine"""
from json import loads
about = { about = {
"website": 'https://deepl.com', "website": 'https://deepl.com',
"wikidata_id": 'Q43968444', "wikidata_id": 'Q43968444',
@ -41,16 +39,14 @@ def request(_query, params):
def response(resp): def response(resp):
results = [] results = []
result = loads(resp.text)
translations = result['translations']
infobox = "<dl>" result = resp.json()
for translation in translations: if not result.get('translations'):
infobox += f"<dd>{translation['text']}</dd>" return results
infobox += "</dl>" translations = [{'text': translation['text']} for translation in result['translations']]
results.append({'answer': infobox}) results.append({'answer': translations[0]['text'], 'answer_type': 'translations', 'translations': translations})
return results return results

21
searx/engines/dictzone.py
View File

@ -3,7 +3,6 @@
Dictzone Dictzone
""" """
from urllib.parse import urljoin
from lxml import html from lxml import html
from searx.utils import eval_xpath from searx.utils import eval_xpath
@ -33,11 +32,10 @@ def request(query, params): # pylint: disable=unused-argument
def response(resp): def response(resp):
results = []
dom = html.fromstring(resp.text) dom = html.fromstring(resp.text)
for k, result in enumerate(eval_xpath(dom, results_xpath)[1:]): translations = []
for result in eval_xpath(dom, results_xpath)[1:]:
try: try:
from_result, to_results_raw = eval_xpath(result, './td') from_result, to_results_raw = eval_xpath(result, './td')
except: # pylint: disable=bare-except except: # pylint: disable=bare-except
@ -49,12 +47,17 @@ def response(resp):
if t.strip(): if t.strip():
to_results.append(to_result.text_content()) to_results.append(to_result.text_content())
results.append( translations.append(
{ {
'url': urljoin(str(resp.url), '?%d' % k), 'text': f"{from_result.text_content()} - {'; '.join(to_results)}",
'title': from_result.text_content(),
'content': '; '.join(to_results),
} }
) )
return results if translations:
result = {
'answer': translations[0]['text'],
'translations': translations,
'answer_type': 'translations',
}
return [result]

13
searx/engines/libretranslate.py
View File

@ -24,7 +24,7 @@ def request(_query, params):
request_url = random.choice(base_url) if isinstance(base_url, list) else base_url request_url = random.choice(base_url) if isinstance(base_url, list) else base_url
params['url'] = f"{request_url}/translate" params['url'] = f"{request_url}/translate"
args = {'source': params['from_lang'][1], 'target': params['to_lang'][1], 'q': params['query']} args = {'source': params['from_lang'][1], 'target': params['to_lang'][1], 'q': params['query'], 'alternatives': 3}
if api_key: if api_key:
args['api_key'] = api_key args['api_key'] = api_key
params['data'] = dumps(args) params['data'] = dumps(args)
@ -42,12 +42,11 @@ def response(resp):
json_resp = resp.json() json_resp = resp.json()
text = json_resp.get('translatedText') text = json_resp.get('translatedText')
from_lang = resp.search_params["from_lang"][1] if not text:
to_lang = resp.search_params["to_lang"][1] return results
query = resp.search_params["query"]
req_url = resp.search_params["req_url"]
if text: translations = [{'text': text}] + [{'text': alternative} for alternative in json_resp.get('alternatives', [])]
results.append({"answer": text, "url": f"{req_url}/?source={from_lang}&target={to_lang}&q={query}"})
results.append({'answer': text, 'answer_type': 'translations', 'translations': translations})
return results return results

41
searx/engines/lingva.py
View File

@ -1,8 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
"""Lingva (alternative Google Translate frontend)""" """Lingva (alternative Google Translate frontend)"""
from json import loads
about = { about = {
"website": 'https://lingva.ml', "website": 'https://lingva.ml',
"wikidata_id": None, "wikidata_id": None,
@ -29,7 +27,7 @@ def request(_query, params):
def response(resp): def response(resp):
results = [] results = []
result = loads(resp.text) result = resp.json()
info = result["info"] info = result["info"]
from_to_prefix = "%s-%s " % (resp.search_params['from_lang'][1], resp.search_params['to_lang'][1]) from_to_prefix = "%s-%s " % (resp.search_params['from_lang'][1], resp.search_params['to_lang'][1])
@ -38,27 +36,40 @@ def response(resp):
if 'definitions' in info: # pylint: disable=too-many-nested-blocks if 'definitions' in info: # pylint: disable=too-many-nested-blocks
for definition in info['definitions']: for definition in info['definitions']:
if 'list' in definition: for item in definition.get('list', []):
for item in definition['list']: for synonym in item.get('synonyms', []):
if 'synonyms' in item:
for synonym in item['synonyms']:
results.append({"suggestion": from_to_prefix + synonym}) results.append({"suggestion": from_to_prefix + synonym})
infobox = "" data = []
for definition in info['definitions']:
for translation in definition['list']:
data.append(
{
'text': result['translation'],
'definitions': [translation['definition']] if translation['definition'] else [],
'examples': [translation['example']] if translation['example'] else [],
'synonyms': translation['synonyms'],
}
)
for translation in info["extraTranslations"]: for translation in info["extraTranslations"]:
for word in translation["list"]: for word in translation["list"]:
infobox += f"<dl><dt>{word['word']}</dt>" data.append(
{
'text': word['word'],
'definitions': word['meanings'],
}
)
for meaning in word["meanings"]: if not data and result['translation']:
infobox += f"<dd>{meaning}</dd>" data.append({'text': result['translation']})
infobox += "</dl>"
results.append( results.append(
{ {
'infobox': result["translation"], 'answer': data[0]['text'],
'content': infobox, 'answer_type': 'translations',
'translations': data,
} }
) )

24
searx/engines/mozhi.py
View File

@ -4,7 +4,6 @@
import random import random
import re import re
from urllib.parse import urlencode from urllib.parse import urlencode
from flask_babel import gettext
about = { about = {
"website": 'https://codeberg.org/aryak/mozhi', "website": 'https://codeberg.org/aryak/mozhi',
@ -35,30 +34,27 @@ def request(_query, params):
def response(resp): def response(resp):
translation = resp.json() translation = resp.json()
infobox = "" data = {'text': translation['translated-text'], 'definitions': [], 'examples': []}
if translation['target_transliteration'] and not re.match( if translation['target_transliteration'] and not re.match(
re_transliteration_unsupported, translation['target_transliteration'] re_transliteration_unsupported, translation['target_transliteration']
): ):
infobox = f"<b>{translation['target_transliteration']}</b>" data['transliteration'] = translation['target_transliteration']
if translation['word_choices']: if translation['word_choices']:
for word in translation['word_choices']: for word in translation['word_choices']:
infobox += f"<dl><dt>{word['word']}: {word['definition']}</dt>" if word.get('definition'):
data['definitions'].append(word['definition'])
if word['examples_target']: for example in word.get('examples_target', []):
for example in word['examples_target']: data['examples'].append(re.sub(r"<|>", "", example).lstrip('- '))
infobox += f"<dd>{re.sub(r'<|>', '', example)}</dd>"
infobox += f"<dd>{re.sub(r'<|>', '', example)}</dd>"
infobox += "</dl>" data['synonyms'] = translation.get('source_synonyms', [])
if translation['source_synonyms']:
infobox += f"<dl><dt>{gettext('Synonyms')}: {', '.join(translation['source_synonyms'])}</dt></dl>"
result = { result = {
'infobox': translation['translated-text'], 'answer': translation['translated-text'],
'content': infobox, 'answer_type': 'translations',
'translations': [data],
} }
return [result] return [result]

26
searx/engines/translated.py
View File

@ -35,18 +35,16 @@ def request(query, params): # pylint: disable=unused-argument
def response(resp): def response(resp):
results = [] json_resp = resp.json()
results.append( text = json_resp['responseData']['translatedText']
{
'url': web_url.format( alternatives = [match['translation'] for match in json_resp['matches'] if match['translation'] != text]
from_lang=resp.search_params['from_lang'][2], translations = [{'text': translation} for translation in [text] + alternatives]
to_lang=resp.search_params['to_lang'][2],
query=resp.search_params['query'], result = {
), 'answer': translations[0]['text'],
'title': '[{0}-{1}] {2}'.format( 'answer_type': 'translations',
resp.search_params['from_lang'][1], resp.search_params['to_lang'][1], resp.search_params['query'] 'translations': translations,
),
'content': resp.json()['responseData']['translatedText'],
} }
)
return results return [result]

38
searx/templates/simple/answerers/translate.html Normal file
View File

@ -0,0 +1,38 @@
<div class="answer-translations">
{% for translation in translations %}
{% if loop.index > 1 %}
<hr />
{% endif %}
<h3>{{ translation.text }}</h3>
{% if translation.transliteration %}
<b>translation.transliteration</b>
{% endif %} {% if translation.definitions %}
<dl>
<dt>{{ _('Definitions') }}</dt>
<ul>
{% for definition in translation.definitions %}
<li>{{ definition }}</li>
{% endfor %}
<ul>
</dl>
{% endif %} {% if translation.examples %}
<dl>
<dt>{{ _('Examples') }}</dt>
<ul>
{% for example in translation.examples %}
<li>{{ example }}</li>
{% endfor %}
</ul>
</dl>
{% endif %} {% if translation.synonyms %}
<dl>
<dt>{{ _('Synonyms') }}</dt>
<ul>
{% for synonym in translation.synonyms %}
<li>{{ synonym }}</li>
{% endfor %}
</ul>
</dl>
{% endif %}
{% endfor %}
</div>

6
searx/templates/simple/results.html
View File

@ -23,6 +23,11 @@
<div id="answers" role="complementary" aria-labelledby="answers-title"><h4 class="title" id="answers-title">{{ _('Answers') }} : </h4> <div id="answers" role="complementary" aria-labelledby="answers-title"><h4 class="title" id="answers-title">{{ _('Answers') }} : </h4>
{%- for answer in answers.values() -%} {%- for answer in answers.values() -%}
<div class="answer"> <div class="answer">
{%- if answer.answer_type == 'translations' -%}
{% with translations=answer.translations %}
{% include 'simple/answerers/translate.html' %}
{% endwith %}
{%- else -%}
<span>{{ answer.answer }}</span> <span>{{ answer.answer }}</span>
{%- if answer.url -%} {%- if answer.url -%}
<a href="{{ answer.url }}" class="answer-url" <a href="{{ answer.url }}" class="answer-url"
@ -31,6 +36,7 @@
{%- endif -%} {%- endif -%}
>{{ urlparse(answer.url).hostname }}</a> >{{ urlparse(answer.url).hostname }}</a>
{% endif -%} {% endif -%}
{%- endif -%}
</div> </div>
{%- endfor -%} {%- endfor -%}
</div> </div>