Merge branch 'master' into boilerplate

Browse Source
This commit is contained in:
Markus Heiser 2019-12-10 13:10:51 +00:00 committed by GitHub
commit 7beb49b1fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 47 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
AUTHORS.rst
View File

@ -1,4 +1,4 @@
Searx was created by Adam Tauber and is maintained by Adam Tauber, Alexandre Flament and Noémi Ványi. Searx was created by Adam Tauber and is maintained by Adam Tauber, Alexandre Flament, Noémi Ványi, @pofilo and Markus Heiser.
Major contributing authors: Major contributing authors:
@ -9,6 +9,8 @@ Major contributing authors:
- @Cqoicebordel - @Cqoicebordel
- Noémi Ványi - Noémi Ványi
- Marc Abonce Seguin @a01200356 - Marc Abonce Seguin @a01200356
- @pofilo
- Markus Heiser @return42
People who have submitted patches/translates, reported bugs, consulted features or People who have submitted patches/translates, reported bugs, consulted features or
generally made searx better: generally made searx better:

21
searx/engines/google.py
View File

@ -107,13 +107,12 @@ images_path = '/images'
supported_languages_url = 'https://www.google.com/preferences?#languages' supported_languages_url = 'https://www.google.com/preferences?#languages'
# specific xpath variables # specific xpath variables
results_xpath = '//div[@class="g"]' results_xpath = '//div[contains(@class, "ZINbbc")]'
url_xpath = './/h3/a/@href' url_xpath = './/div[@class="kCrYT"][1]/a/@href'
title_xpath = './/h3' title_xpath = './/div[@class="kCrYT"][1]/a/div[1]'
content_xpath = './/span[@class="st"]' content_xpath = './/div[@class="kCrYT"][2]//div[contains(@class, "BNeawe")]//div[contains(@class, "BNeawe")]'
content_misc_xpath = './/div[@class="f slp"]' suggestion_xpath = '//div[contains(@class, "ZINbbc")][last()]//div[@class="rVLSBd"]/a//div[contains(@class, "BNeawe")]'
suggestion_xpath = '//p[@class="_Bmc"]' spelling_suggestion_xpath = '//div[@id="scc"]//a'
spelling_suggestion_xpath = '//a[@class="spell"]'
# map : detail location # map : detail location
map_address_xpath = './/div[@class="s"]//table//td[2]/span/text()' map_address_xpath = './/div[@class="s"]//table//td[2]/span/text()'
@ -199,10 +198,6 @@ def request(query, params):
params['headers']['Accept-Language'] = language + ',' + language + '-' + country params['headers']['Accept-Language'] = language + ',' + language + '-' + country
params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
# Force Safari 3.1 on Mac OS X (Leopard) user agent to avoid loading the new UI that Searx can't parse
params['headers']['User-Agent'] = ("Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_4)"
"AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.20.1")
params['google_hostname'] = google_hostname params['google_hostname'] = google_hostname
return params return params
@ -274,9 +269,7 @@ def response(resp):
content = extract_text_from_dom(result, content_xpath) content = extract_text_from_dom(result, content_xpath)
if content is None: if content is None:
continue continue
content_misc = extract_text_from_dom(result, content_misc_xpath)
if content_misc is not None:
content = content_misc + "<br />" + content
# append result # append result
results.append({'url': url, results.append({'url': url,
'title': title, 'title': title,

115
tests/unit/engines/test_google.py
View File

@ -58,93 +58,50 @@ class TestGoogleEngine(SearxTestCase):
self.assertEqual(google.response(response), []) self.assertEqual(google.response(response), [])
html = """ html = """
<div class="g"> <div class="ZINbbc xpd O9g5cc uUPGi">
<h3 class="r"> <div>
<a href="http://this.should.be.the.link/"> <div class="kCrYT">
<a href="/url?q=http://this.should.be.the.link/">
<div class="BNeawe">
<b>This</b> is <b>the</b> title <b>This</b> is <b>the</b> title
</div>
<div class="BNeawe">
http://website
</div>
</a> </a>
</h3>
<div class="s">
<div class="kv" style="margin-bottom:2px">
<cite>
<b>test</b>.psychologies.com/
</cite>
<div class="_nBb">‎
<div style="display:inline" onclick="google.sham(this);" aria-expanded="false"
aria-haspopup="true" tabindex="0" data-ved="0CBUQ7B0wAA">
<span class="_O0">
</span>
</div> </div>
<div style="display:none" class="am-dropdown-menu" role="menu" tabindex="-1"> <div class="kCrYT">
<ul> <div>
<li class="_Ykb"> <div class="BNeawe">
<a class="_Zkb" href="http://www.google.fr/url?url=http://webcache.googleusercontent <div>
.com/search%3Fcache:R1Z_4pGXjuIJ:http://test.psychologies.com/"> <div class="BNeawe">
En cache
</a>
</li>
<li class="_Ykb">
<a class="_Zkb" href="/search?safe=off&amp;q=related:test.psy.com/">
Pages similaires
</a>
</li>
</ul>
</div>
</div>
</div>
<span class="st">
This should be the content. This should be the content.
</div>
</div>
</div>
</div>
</div>
</div>
</p>
<div class="ZINbbc xpd O9g5cc uUPGi">
<div>
<div class="kCrYT">
<span>
<div class="BNeawe">
Related searches
</div>
</span> </span>
<br> </div>
<div class="osl">‎ <div class="rVLSBd">
<a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/"> <a>
Test Personnalité <div>
</a> - ‎ <div class="BNeawe">
<a href="http://www.google.fr/url?url=http://test.psychologies.com/test/"> suggestion title
Tests - Moi </div>
</a> - ‎ </div>
<a href="http://www.google.fr/url?url=http://test.psychologies.com/test/tests-couple">
Test Couple
</a>
- ‎
<a href="http://www.google.fr/url?url=http://test.psychologies.com/tests/tests-amour">
Test Amour
</a> </a>
</div> </div>
</div> </div>
</div>
<div class="g">
<h3 class="r">
<a href="http://www.google.com/images?q=toto">
<b>This</b>
</a>
</h3>
</div>
<div class="g">
<h3 class="r">
<a href="http://www.google.com/search?q=toto">
<b>This</b> is
</a>
</h3>
</div>
<div class="g">
<h3 class="r">
<a href="€">
<b>This</b> is <b>the</b>
</a>
</h3>
</div>
<div class="g">
<h3 class="r">
<a href="/url?q=url">
<b>This</b> is <b>the</b>
</a>
</h3>
</div>
<p class="_Bmc" style="margin:3px 8px">
<a href="/search?num=20&amp;safe=off&amp;q=t&amp;revid=1754833769&amp;sa=X&amp;ei=-&amp;ved=">
suggestion <b>title</b>
</a>
</p> </p>
""" """
response = self.mock_response(html) response = self.mock_response(html)